publications

@inproceedings{cvpr24-TeachCLIP,

title = {Holistic Features are almost Sufficient for Text-to-Video Retrieval},

author = {Kaibin Tian and Ruixiang Zhao and Zijie Xin and Bangxiang Lan and Xirong Li},

url = {https://lixirong.net/pub/cvpr2024-TeachCLIP.pdf},

year  = {2024},

date = {2024-06-19},

urldate = {2024-06-19},

booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{coling24-sea,

title = {Tackling Long Code Search with Splitting, Encoding, and Aggregating},

author = {Fan Hu and Yanlin Wang and Lun Du and Hongyu Zhang and Shi Han and Dongmei Zhang and Xirong Li},

year  = {2024},

date = {2024-05-20},

urldate = {2024-05-20},

booktitle = {The 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icassp2024-cliprerank,

title = {CLIPRerank: An Extremely Simple Method for Improving Ad-hoc Video Search},

author = {Aozhu Chen and Fangming Zhou and Ziyuan Wang and Xirong Li},

year  = {2024},

date = {2024-04-14},

urldate = {2024-04-14},

booktitle = {ICASSP},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{bibm23-sda,

title = {Supervised Domain Adaptation for Recognizing Retinal Diseases from Wide-Field Fundus Images},

author = {Qijie Wei and Jingyuan Yang and Bo Wang and Jinrui Wang and Jianchun Zhao and Xinyu Zhao and Sheng Yang and Niranchana Manivannan and Youxin Chen and Dayong Ding and Jing Zhou and Xirong Li},

year  = {2023},

date = {2023-12-05},

urldate = {2023-12-05},

booktitle = {IEEE International Conference on Bioinformatics & Biomedicine (BIBM)},

abstract = {This paper addresses the emerging task of recognizing multiple retinal diseases from wide-field (WF) and ultra-wide-field (UWF) fundus images. For an effective use of existing large amount of labeled color fundus photo (CFP) data and the relatively small amount of WF and UWF data, we propose a supervised domain adaptation method named Cross-domain Collaborative Learning (CdCL). Inspired by the success of fixed-ratio based mixup in unsupervised domain adaptation, we re-purpose this strategy for the current task. Due to the intrinsic disparity between the field-of-view of CFP and WF/UWF images, a scale bias naturally exists in a mixup sample that the anatomic structure from a CFP image will be considerably larger than its WF/UWF counterpart. The CdCL method resolves the issue by Scale-bias Correction, which employs Transformers for producing scale-invariant features. As demonstrated by extensive experiments on multiple datasets covering both WF and UWF images, the proposed method compares favorably against a number of competitive baseline.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm23-ChinaOpen,

title = {ChinaOpen: A Dataset for Open-world Multimodal Learning},

author = {Aozhu Chen and Ziyuan Wang and Chengbo Dong and Kaibin Tian and Ruixiang Zhao and Xun Liang and Zhanhui Kang and Xirong Li},

year  = {2023},

date = {2023-10-29},

booktitle = {ACM Multimedia},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{iccv2023-GeoFormer,

title = {Geometrized Transformer for Self-Supervised Homography Estimation},

author = {Jiazhen Liu and Xirong Li},

year  = {2023},

date = {2023-10-03},

booktitle = {International Conference on Computer Vision (ICCV)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{iccv23-saflnet,

title = {SAFL-Net: Semantic-Agnostic Feature Learning Network with Auxiliary Plugins for Image Manipulation Detection},

author = {Zhihao Sun and Haoran Jiang and Danding Wang and Xirong Li and Juan Cao},

year  = {2023},

date = {2023-10-02},

urldate = {2023-10-02},

booktitle = {International Conference on Computer Vision (ICCV)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icassp2023-tha,

title = {Towards Making a Trojan-horse Attack on Text-to-Image Retrieval},

author = {Fan Hu and Aozhu Chen and Xirong Li},

year  = {2023},

date = {2023-06-04},

booktitle = {IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP)},

abstract = {While deep learning based image retrieval is reported to be vulnerable to adversarial attacks, existing works are mainly on image-to-image retrieval with their attacks performed at the front end via query modification. By contrast, we present in this paper the first study about a threat that occurs at the back end of a text-to-image retrieval (T2IR) system. Our study is motivated by the fact that the image collection indexed by the system will be regularly updated due to the arrival of new images from various sources such as web crawlers and advertisers. With malicious images indexed, it is possible for an attacker to indirectly interfere with the retrieval process, letting users see certain images that are completely irrelevant w.r.t. their queries. We put this thought into practice by proposing a novel Trojan-horse attack (THA). In particular, we construct a set of Trojan-horse images by first embedding word-specific adversarial information into a QR code and then putting the code on benign advertising images. A proof-of-concept evaluation, conducted on two popular T2IR datasets (Flickr30k and MS-COCO), shows the effectiveness of the proposed THA in a white-box mode.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{eccv2022-laff-video-retrieval,

title = {Lightweight Attentional Feature Fusion: A New Baseline for Text-to-Video Retrieval},

author = {Fan Hu and Aozhu Chen and Ziyue Wang and Fangming Zhou and Jianfeng Dong and Xirong Li},

year  = {2022},

date = {2022-10-23},

booktitle = {European Conference on Computer Vision (ECCV)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{eccv2022-SuperRetina,

title = {Semi-Supervised Keypoint Detector and Descriptor for Retinal Image Matching},

author = {Jiazhen Liu and Xirong Li and Qijie Wei and Jie Xu and Dayong Ding},

year  = {2022},

date = {2022-10-23},

booktitle = {European Conference on Computer Vision (ECCV)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm22-negation-learning,

title = {Learn to Understand Negation in Video Retrieval},

author = {Ziyue Wang and Aozhu Chen and Fan Hu and Xirong Li},

year  = {2022},

date = {2022-10-10},

urldate = {2022-10-10},

booktitle = {ACM Multimedia},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2022-prvr,

title = {Partially Relevant Video Retrieval},

author = {Jianfeng Dong and Xiaoke Chen and Minsong Zhang and Xun Yang and Shujie Chen and Xirong Li and Xun Wang},

year  = {2022},

date = {2022-10-10},

urldate = {2022-10-10},

booktitle = {ACM Multimedia},

journal = {ACM Multimedia},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icmr22-oct-ssod,

title = {Lesion Localization in OCT by Semi-Supervised Object Detection},

author = {Yue Wu and Yang Zhou and Jianchun Zhao and Jingyuan Yang and Weihong Yu and Youxin Chen and Xirong Li},

doi = {10.1145/3512527.3531418},

year  = {2022},

date = {2022-06-27},

booktitle = {ACM International Conference on Multimedia Retrieval (ICMR)},

abstract = {Over 300 million people worldwide are affected by various retinal diseases. By noninvasive Optical Coherence Tomography (OCT) scans, a number of abnormal structural changes in the retina, namely retinal lesions, can be identified. Automated lesion localization in OCT is thus important for detecting retinal diseases at their early stage. To conquer the lack of manual annotation for deep supervised learning, this paper presents a first study on utilizing semi-supervised object detection (SSOD) for lesion localization in OCT images. To that end, we develop a taxonomy to provide a unified and structured viewpoint of the current SSOD methods, and consequently identify key modules in these methods. To evaluate the influence of these modules in the new task, we build OCT-SS, a new dataset consisting of over 1k expert-labeled OCT B-scan images and over 13k unlabeled B-scans. Extensive experiments on OCT-SS identify Unbiased Teacher (UnT) as the best current SSOD method for lesion localization. Moreover, we improve over this strong baseline, with mAP increased from 49.34 to 50.86.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm21-mm-mil,

title = {Multi-Modal Multi-Instance Learning for Retinal Disease Recognition},

author = {Xirong Li and Yang Zhou and Jie Wang and Hailan Lin and Jianchun Zhao and Dayong Ding and Weihong Yu and Youxin Chen},

url = {pub/mm2021-mm-mil.pdf},

doi = {10.1145/3474085.3475418},

year  = {2021},

date = {2021-10-20},

urldate = {2021-10-20},

booktitle = {ACM Multimedia},

abstract = {This paper attacks an emerging challenge of multi-modal retinal disease recognition. Given a multi-modal case consisting of a color fundus photo (CFP) and an array of OCT B-scan images acquired during an eye examination, we aim to build a deep neural network that recognizes multiple vision-threatening diseases for the given case. As the diagnostic efficacy of CFP and OCT is disease-dependent, the network's ability of being both selective and interpretable is important. Moreover, as both data acquisition and manual labeling are extremely expensive in the medical domain, the network has to be relatively lightweight for learning from a limited set of labeled multi-modal samples. Prior art on retinal disease recognition focuses either on a single disease or on a single modality, leaving multi-modal fusion largely underexplored. We propose in this paper Multi-Modal Multi-Instance Learning (MM-MIL) for selectively fusing CFP and OCT modalities. Its lightweight architecture (as compared to current multi-head attention modules) makes it suited for learning from relatively small-sized datasets. For an effective use of MM-MIL, we propose to generate a pseudo sequence of CFPs by over sampling a given CFP. The benefits of this tactic include well balancing instances across modalities, increasing the resolution of the CFP input, and finding out regions of the CFP most relevant with respect to the final diagnosis. Extensive experiments on a real-world dataset consisting of 1,206 multi-modal cases from 1,193 eyes of 836 subjects demonstrate the viability of the proposed model.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm21-videocap,

title = {Multi-Level Visual Representation with Semantic-Reinforced Learning for Video Captioning},

author = {Chengbo Dong and Xinru Chen and Aozhu Chen and Fan Hu and Zihan Wang and Xirong Li},

year  = {2021},

date = {2021-10-20},

booktitle = {ACM Multimedia (Grand Challenge Track)},

abstract = {This paper describes our bronze-medal solution for the video captioning task of the ACMMM2021 Pre-Training for Video Understanding Challenge. We depart from the Bottom-Up-Top-Down model, with technical improvements on both video content encoding and caption decoding. For encoding, we propose to extract multi-level video features that describe holistic scenes and fine-grained key objects, respectively. The scene-level and object-level features are enhanced separately by multi-head self-attention mechanisms before feeding them into the decoding module. Towards generating content-relevant and human-like captions, we train our network end-to-end by semantic-reinforced learning. Finally, in order to select the best caption from captions produced by different models, we perform caption reranking by cross-modal matching between a given video and each candidate caption. Both internal experiments on the MSR-VTT test set and external evaluations by the challenge organizers justify the viability of the proposed solution.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{nokey,

title = {Improving Fake News Detection by Using an Entity-enhanced Framework to Fuse Diverse Multimodal Clues},

author = {Peng Qi and Juan Cao and Xirong Li and Huan Liu and Qiang Sheng and Xiaoyue Mi and Qin He and Yongbiao Lv and Chenyang Guo and Yingchao Yu},

year  = {2021},

date = {2021-10-20},

booktitle = {ACM Multimedia (Industrial Track)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{viral21-avs,

title = {What Matters for Ad-hoc Video Search? A Large-scale Evaluation on TRECVID},

author = {Aozhu Chen and Fan Hu and Zihan Wang and Fangming Zhou and Xirong Li},

year  = {2021},

date = {2021-10-16},

urldate = {2021-10-16},

booktitle = {The 2nd International Workshop on Video Retrieval Methods and Their Limits (ViRal'21@ICCV 2021)},

abstract = {For quantifying progress in Ad-hoc Video Search (AVS), the annual TRECVID AVS task is an important international evaluation. Solutions submitted by the task participants vary in terms of their choices of cross-modal matching models, visual features and training data. As such, what one may conclude from the evaluation is at a high level that is insufficient to reveal the influence of the individual components. In order to bridge the gap between the current solution-level comparison and the desired component-wise comparison, we propose in this paper a large-scale and systematic evaluation on TRECVID. By selected combinations of state-of-the-art matching models, visual features and (pre-)training data, we construct a set of 25 different solutions and evaluate them on the TRECVID AVS tasks 2016--2020. The presented evaluation helps answer the key question of what matters for AVS. The resultant observations and learned lessons are also instructive for developing novel AVS solutions.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{iccv21-mvssnet,

title = {Image Manipulation Detection by Multi-View Multi-Scale Supervision},

author = {Xinru Chen and Chengbo Dong and Jiaqi Ji and Juan Cao and Xirong Li},

url = {pub/iccv2021-mvssnet.pdf},

year  = {2021},

date = {2021-10-11},

urldate = {2021-10-11},

booktitle = {International Conference on Computer Vision (ICCV) },

abstract = {The key challenge of image manipulation detection is how to learn generalizable features that are sensitive to manipulations in novel data, whilst specific to prevent false alarms on authentic images. Current research emphasizes the sensitivity, with the specificity overlooked. In this paper we address both aspects by multi-view feature learning and multi-scale supervision. By exploiting noise distribution and boundary artifact surrounding tampered regions, the former aims to learn semantic-agnostic and thus more generalizable features. The latter allows us to learn from authentic images which are nontrivial to taken into account by current semantic segmentation network based methods. Our thoughts are realized by a new network which we term MVSS-Net. Extensive experiments on five benchmark sets justify the viability of MVSS-Net for both pixel-level and image-level manipulation detection.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{acl21-fact-check,

title = {Article Reranking by Memory-enhanced Key Sentence Matching for Detecting Previously Fact-checked Claims},

author = {Qiang Sheng and Juan Cao and Xueyao Zhang and Xirong Li and Lei Zhong},

year  = {2021},

date = {2021-08-01},

booktitle = {The Joint Conference of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (ACL-IJCNLP 2021)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{web2021-fake-news-detection,

title = {Mining Dual Emotion for Fake News Detection},

author = {Xueyao Zhang and Juan Cao and Xirong Li and Qiang Sheng and Lei Zhong and Kai Shu},

url = {http://lixirong.net/pub/www2021_fake_news_detection.pdf},

year  = {2021},

date = {2021-04-19},

booktitle = {The Web Conference 2021 (WWW)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icpr20-rop,

title = {Deep Multiple Instance Learning with Spatial Attention for ROP Case Classification, Instance Selection and Abnormality Localization  },

author = {Xirong Li and Wencui Wan and Yang Zhou and Jianchun Zhao and Qijie Wei and Junbo Rong and Pengyi Zhou and Limin Xu and Lijuan Lang and Yuying Liu and Chengzhi Niu and Dayong Ding and Xuemin Jin},

url = {http://lixirong.net/pub/icpr2020-rop.pdf},

year  = {2020},

date = {2020-12-31},

booktitle = {25th International Conference on Pattern Recognition (ICPR2020)},

abstract = {This paper tackles automated screening of Retinopathy of Prematurity (ROP), one of the most common causes of visual loss in childhood. Clinically, ROP screening per case requires multiple color fundus images capturing different zones of the premature retina. A desirable model shall not only make a decision at the case level, but also pinpoint which instances and what part of the instances are responsible for the decision. This paper makes the first attempt to accomplish three tasks, i.e, ROP case classification, instance selection and abnormality localization in a unified framework. To that end, we propose a new model that effectively combines instance-attention based deep multiple instance learning (MIL) and spatial attention (SA). The propose model, which we term MIL-SA, identifies positive instances in light of

their contributions to case-level decision. Meanwhile, abnormal regions in the identified instances are automatically localized by the SA mechanism. Moreover, MIL-SA is learned from case-level binary labels exclusively, and in an end-to-end manner. Experiments on a large clinical dataset of 2,186 cases with 11,053 fundus images show the viability of the proposed model for all the three tasks. },

note = {oral},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icpr20-lesion-net,

title = {Learn to Segment Retinal Lesions and Beyond },

author = {Qijie Wei and Xirong Li and Weihong Yu and xiao Zhang and Yongpeng Zhang and Bojie Hu and Bin Mo and Di Gong and Ning Chen and Dayong Ding and Youxin Chen},

url = {http://lixirong.net/pub/icpr2020-lesion.pdf},

year  = {2020},

date = {2020-12-31},

booktitle = {25th International Conference on Pattern Recognition (ICPR2020)},

abstract = {Towards automated retinal screening, this paper makes an endeavor to simultaneously achieve pixel-level retinal lesion segmentation and image-level disease classification. Such a multi-task approach is crucial for accurate and clinically interpretable disease diagnosis. Prior art is insufficient due to three challenges, i.e., lesions lacking objective boundaries, clinical importance of lesions irrelevant to their size, and the lack of one-to-one correspondence between lesion and disease classes. This paper attacks the three challenges in the context of diabetic retinopathy (DR) grading. We propose Lesion-Net, a new variant of fully convolutional networks, with its expansive path re-designed to tackle the first challenge. A dual Dice loss that leverages both semantic segmentation and image classification losses is introduced to resolve the second challenge. Lastly, we build a multi-task network that employs Lesion-Net as a side-attention branch for both DR grading and result interpretation. A set of 12K fundus images is manually segmented by 45 ophthalmologists for 8 DR-related lesions, resulting in 290K manual segments in total. Extensive experiments on this large-scale dataset show that our proposed approach surpasses the prior art for multiple tasks including lesion segmentation, lesion classification and DR grading. },

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mmasia20-anno-free,

title = {Towards Annotation-Free Evaluation of Cross-Lingual Image Captioning},

author = {Aozhu Chen and Xinyi Huang and Hailan Lin and Xirong Li},

url = {http://lixirong.net/pub/mmasia2020-eval-cap.pdf},

doi = {10.1145/3444685.3446322},

year  = {2020},

date = {2020-12-31},

booktitle = {ACM Multimedia Asia (MMAsia)},

abstract = {Cross-lingual image captioning, with its ability to caption an unlabelled image in a target language other than English, is an emerging topic in the multimedia field. A large-scale evaluation of the resultant models typically needs thousands of test images associated with manually written captions in the same language. In order to save the precious human resource from re-writing reference sentences per target language, in this paper we make a brave attempt towards annotation-free evaluation of cross-lingual image captioning. Depending on whether we assume the availability of English references, two scenarios are investigated. For the first scenario with the references available, we propose two metrics, i.e., WMDRel and CLinRel. WMDRel measures the semantic relevance between a model-generated caption and machine translation of an English reference using their Word Mover Distance. By projecting both captions into a deep visual feature space, CLinRel is a visual-oriented cross-lingual relevance measure. As for the second scenario, which has zero reference and is thus more challenging, we propose CMedRel to compute a cross-media relevance between the generated caption and the image content, in the same visual feature space as used by CLinRel. We have conducted a number of experiments to evaluate the effectiveness of the three proposed metrics. The combination of WMDRel, CLinRel and CMedRel has a Spearman’s rank correlation of 0.952 with the sum of BLEU-4, METEOR, ROUGE-L and CIDEr, four standard metrics computed using references in the target language. CMedRel alone has a Spearman’s rank correlation of 0.786 with the standard metrics. The promising results show high potential of the new metrics for evaluation with no need of references in the target language.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{tv20-rucmm,

title = {Renmin University of China at TRECVID 2020: Sentence Encoder Assembly for Ad-hoc Video Search},

author = {Xirong Li and Fangming Zhou and Aozhu Chen},

year  = {2020},

date = {2020-12-03},

booktitle = {TRECVID 2020 Workshop},

abstract = {In this paper we summarize our TRECVID 2020 video retrieval experiments. We participated in the Ad-hoc Video Search (AVS) task with fully deep learning based solutions. Our solutions contain two deep models and their variants. One is W2VV++ that vectorizes a given textual query by concatenating the output of multiple sentence encoders such as bag-of-words (BoW), word2vec and GRU. The other is a newly developed model, which we term Sentence Encoder Assembly (SEA). The novelty of the SEA model is two-fold. First, different from the prior art that uses only a single common space, SEA supports text-video matching in multiple encoder-specific common spaces. Such a property prevents the matching from being dominated by a specific encoder that produces an encoding vector much longer than other encoders. Second, in order to explore complementarities among the individual common spaces, we propose to train SEA by multi-space multi-loss learning. We exploit MSR-VTT and TGIF as training data. For video representation, we use pre-trained ResNet-152 and ResNeXt-101 to extract frame-level features, and C3D to extract segment-level features. Video-level features are obtained by mean pooling. Using SEA alone obtains a mean infAP of 0.236 for the 2020 task. Having SEA pre-trained on the Google’s Conceptual Captions dataset is helpful, obtaining a higher infAP of 0.251. We again find late average fusion of dis- tinct models (consisting of SEA and W2VV++ trained in varied settings) beneficial, obtaining the best infAP of 0.269 among our four submissions, and ranked at the second place teamwise.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm20-video-retrieval,

title = {A W2VV++ Case Study with Automated and Interactive Text-to-Video Retrieval},

author = {Jakub Lokoč and Tomáš Souček and Patrik Veselý and František Mejzlík and Jiaqi Ji and Chaoxi Xu and Xirong Li},

url = {http://lixirong.net/pub/mm2020-lokoc.pdf},

doi = {10.1145/3394171.3414002},

year  = {2020},

date = {2020-10-12},

booktitle = {ACM Multimedia},

abstract = {As reported by respected evaluation campaigns focusing both on automated and interactive video search approaches, deep learning started to dominate the video retrieval area. However, the results are still not satisfactory for many types of search tasks focusing on high recall. To report on this challenging problem, we present two orthogonal task-based performance studies centered around the state-of-the-art W2VV++ query representation learning model for video retrieval. First, an ablation study is presented to investigate which components of the model are effective in two types of benchmark tasks focusing on high recall. Second, interactive search scenarios from the Video Browser Showdown are analyzed for two winning prototype systems providing additional querying and visualization components for the incorporated W2VV++ text search model. The analysis of collected logs demonstrates that even with state-of-the-art text search video retrieval models, it is still auspicious to integrate users into the search process for tasks, where high recall is essential.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icmr20-icap,

title = {iCap: Interactive Image Captioning with Predictive Text},

author = {Zhengxiong Jia and Xirong Li},

url = {pub/icmr2020-icap.pdf},

doi = {10.1145/3372278.3390697},

year  = {2020},

date = {2020-06-08},

booktitle = {ACM International Conference on Multimedia Retrieval (ICMR)},

abstract = {In this paper we study a brand new topic of interactive image captioning with human in the loop. Different from automated image captioning where a given test image is the sole input in the inference stage, we have access to both the test image and a sequence of (incomplete) user-input sentences in the interactive scenario. We formulate the problem as Visually Conditioned Sentence Completion (VCSC). For VCSC, we propose ABD-Cap, asynchronous bidirectional decoding for image caption completion. With ABD-Cap as the core module, we build iCap, a web-based interactive image captioning system capable of predicting new text with respect to live input from a user. A number of experiments covering both automated evaluations and real user studies show the viability of our proposals.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{trecvid19-rucmm,

title = {Renmin University of China and Zhejiang Gongshang University at TRECVID 2019: Learn to Search and Describe Videos},

author = {Xirong Li and Jinde Ye and Chaoxi Xu and Shanjinwen Yun and Leimin Zhang and Xun Wang and Rui Qian and Jianfeng Dong},

url = {pub/trecvid2019-rucmm.pdf},

year  = {2019},

date = {2019-11-12},

booktitle = {TRECVID 2019 Workshop},

abstract = {In this paper we summarize our TRECVID 2019 video retrieval experiments. We participated in two tasks: Ad-hoc Video Search (AVS) and Video-to-Text (VTT). For the AVS task, we develop our solutions based on two deep learn- ing models, i.e. the W2VV++ network and the Dual Encoding Network. For the VTT Matching and Rank- ing subtask, our entry is also based on the W2VV++ and Dual Encoding Networks. For the VTT Description Generation subtask, we enhance the classical encoder-decoder model with multi-level video encoding and attribute prediction. The 2019 edition of the TRECVID benchmark has been a fruitful participation for our joint-team. Our runs are ranked at the second place for AVS and VTT Matching and Ranking tasks and the third place for the VTT Description Generation subtask in terms of the ciderD criterion.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2019-w2vvpp,

title = {W2VV++: Fully Deep Learning for Ad-hoc Video Search},

author = {Xirong Li and Chaoxi Xu and Gang Yang and Zhineng Chen and Jianfeng Dong},

url = {pub/mm2019-w2vvpp.pdf},

doi = {10.1145/3343031.3350906},

year  = {2019},

date = {2019-10-21},

booktitle = { ACM Multimedia},

abstract = {Ad-hoc video search (AVS) is an important yet challenging problem in multimedia retrieval. Different from previous concept-based methods, we propose an end-to-end deep learning method for query representation learning. The proposed method requires no concept modeling, matching and selection. The backbone of our method is the proposed W2VV++ model, a super version of Word2VisualVec (W2VV) previously developed for visual-to-text matching. W2VV++ is obtained by tweaking W2VV with a better sentence encoding strategy and an improved triplet ranking loss. With these simple changes, W2VV++ brings in a substantial improvement in performance. As our participation in the TRECVID 2018 AVS task and retrospective experiments on the TRECVID 2016 and 2017 data show, our best single model, with an overall inferred average precision (infAP) of 0.157, outperforms the state-of-the-art. The performance can be further boosted by model ensemble using late average fusion, reaching a higher infAP of 0.163. With W2VV++, we establish a new baseline for ad-hoc video search.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{miccai19-cataract,

title = {Fully Deep Learning for Slit-lamp Photo based Nuclear Cataract Grading},

author = {Chaoxi Xu and Xiangjia Zhu and Wenwen He and Yi Lu and Xixi He and Zongjiang Shang and Jun Wu and Keke Zhang and Yinglei Zhang and Xianfang Rong and Zhennan Zhao and Lei Cai and Dayong Ding and Xirong Li},

url = {pub/miccai2019-cataract.pdf},

doi = {10.1007/978-3-030-32251-9_56},

year  = {2019},

date = {2019-10-13},

booktitle = {International Conference on Medical Image Computing and Computer Assisted Intervention (MICCAI)},

note = {early accept},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{miccai19-amd,

title = {Two-Stream CNN with Loose Pair Training for Multi-modal AMD Categorization},

author = {Weisen Wang and Zhiyan Xu and Weihong Yu and Jianchun Zhao and Jingyuan Yang and Feng He and Zhikun Yang and Di Chen and Dayong Ding and Youxin Chen and Xirong Li},

url = {https://link.springer.com/chapter/10.1007%2F978-3-030-32239-7_18},

doi = {10.1007/978-3-030-32239-7_18},

year  = {2019},

date = {2019-10-13},

booktitle = {International Conference on Medical Image Computing and Computer Assisted Intervention (MICCAI)},

note = {early accept},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mlmi2019-uwf,

title = {Joint Localization of Optic Disc and Fovea in Ultra-Widefield Fundus Images},

author = {Zhuoya Yang and Xirong Li and Xixi He and Dayong Ding and Yanting Wang and Fangfang Dai and Xuemin Jin },

url = {http://lixirong.net/pub/mlmi2019-uwf.pdf},

doi = {10.1007/978-3-030-32692-0_52},

year  = {2019},

date = {2019-10-13},

booktitle = {The 10th International Workshop on Machine Learning in Medical Imaging (MLMI)},

note = {in conjunction with MICCAI 2019},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{cvpr19-zevr,

title = {Dual Encoding for Zero-Example Video Retrieval},

author = {Jianfeng Dong and Xirong Li and Chaoxi Xu and Shouling Ji and Yuan He and Gang Yang and Xun Wang},

url = {/pub/cvpr2019-dense-encoding.pdf},

year  = {2019},

date = {2019-06-15},

booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},

abstract = {This paper attacks the challenging problem of zero- example video retrieval. In such a retrieval paradigm, an end user searches for unlabeled videos by ad-hoc queries described in natural language text with no visual example provided. Given videos as sequences of frames and queries as sequences of words, an effective sequence-to-sequence cross-modal matching is required. The majority of exist- ing methods are concept based, extracting relevant concepts from queries and videos and accordingly establish- ing associations between the two modalities. In contrast, this paper takes a concept-free approach, proposing a dual deep encoding network that encodes videos and queries into powerful dense representations of their own. Dual encoding is conceptually simple, practically effective and end- to-end. As experiments on three benchmarks, i.e. MSR- VTT, TRECVID 2016 and 2017 Ad-hoc Video Search show, the proposed solution establishes a new state-of-the-art for zero-example video retrieval.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{trecvid2018-rucmm,

title = {Renmin University of China and Zhejiang Gongshang University at TRECVID 2018: Deep Cross-Modal Embeddings for Video-Text Retrieval},

author = {Xirong Li and Jianfeng Dong and Chaoxi Xu and Jing Cao and Xun Wang and Gang Yang},

url = {pub/trecvid2018-rucmm.pdf},

year  = {2018},

date = {2018-11-13},

booktitle = {TRECVID 2018 Workshop},

abstract = {In this paper we summarize our TRECVID 2018 video retrieval experiments. We participated in two tasks: Ad-hoc Video Search (AVS) and Video-to-Text (VTT) Matching and Ranking. For the AVS task, we develop our solutions based on W2VV++, a super version of Word2VisualVec (W2VV). For the VTT task, our entry is built on the top of a recently proposed dual dense encoding network, which encodes an input, let it be a video or a natural language sentence, in a dense manner. The 2018 edition of the TRECVID benchmark has been a fruitful participation for our joint-team, resulting in the best overall result for both AVS and VTT tasks. Retrospective experiments show that our ad-hoc video search system, used as is, also outperforms the previous best results of the TRECVID 2016 and 2017 AVS tasks. We have released feature data at https://github.com/li-xirong/avs},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2017-fluency,

title = {Fluency-Guided Cross-Lingual Image Captioning},

author = {Weiyu Lan and Xirong Li and Jianfeng Dong},

url = {/pub/mm2017-weiyu.pdf},

doi = {10.1145/3123266.3123366},

year  = {2017},

date = {2017-10-23},

booktitle = {ACM Multimedia},

abstract = {Image captioning has so far been explored mostly in English, as most available datasets are in this language. However, the application of image captioning should not be restricted by language. Only few studies have been conducted for image captioning in a cross-lingual setting. Different from these works that manually build a dataset for a target language, we aim to learn a cross-lingual captioning model fully from machine-translated sentences. To conquer the lack of fluency in the translated sentences, we propose in this paper a fluency-guided learning framework. The framework comprises a module to automatically estimate the fluency of the sentences and another module to utilize the estimated fluency scores to effectively train an image captioning model for the target language. As experiments on two bilingual (English-Chinese) datasets show, our approach improves both fluency and relevance of the generated captions in Chinese, but without using any manually written sentences from the target language.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2016-v2t,

title = {Early Embedding and Late Reranking for Video Captioning},

author = {Jianfeng Dong and Xirong Li and Weiyu Lan and Yujia Huo and Cees G. M. Snoek},

url = {http://lixirong.net/pub/mm2016-video2text.pdf},

doi = {10.1145/2964284.2984064},

year  = {2016},

date = {2016-10-15},

urldate = {2016-10-15},

booktitle = {ACM Multimedia},

abstract = {This paper describes our solution for the MSR Video to Language Challenge. We start from the popular ConvNet + LSTM model, which we extend with two novel modules. One is early embedding, which enriches the current low-level input to LSTM by tag embeddings. The other is late reranking, for re-scoring generated sentences in terms of their relevance to a specific video. The modules are inspired by recent works on image captioning, repurposed and redesigned for video. As experiments on the MSR-VTT validation set show, the joint use of these two modules add a clear improvement over a non-trivial ConvNet + LSTM baseline under four performance metrics. The viability of the proposed solution is further confirmed by the blind test by the organizers. Our system is ranked at the 4th place in terms of overall performance, while scoring the best CIDEr-D, which measures the human-likeness of generated captions.},

note = {Grand Challenge Award},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2016-videoviolence,

title = {Detecting Violence in Video using Subclasses},

author = {Xirong Li and Yujia Huo and Qin Jin and Jieping Xu},

url = {http://lixirong.net/pub/mm2016-vsd.pdf},

doi = {10.1145/2964284.2967289},

year  = {2016},

date = {2016-10-01},

booktitle = {ACM Multimedia},

abstract = {This paper attacks the challenging problem of violence detection in videos. Different from existing works focusing on combining multi-modal features, we go one step further by adding and exploiting subclasses visually related to violence. We enrich the MediaEval 2015 violence dataset by manually labeling violence videos with respect to the subclasses. Such fine-grained annotations not only help understand what have impeded previous efforts on learning to fuse the multi-modal features, but also enhance the generalization ability of the learned fusion to novel test data. The new subclass based solution, with AP of 0.303 and P100 of 0.55 on the MediaEval 2015 test set, outperforms the state-of-the-art. Notice that our solution does not require fine-grained annotations on the test set, so it can be directly applied on novel and fully unlabeled videos. Interestingly, our study shows that motion related features (MBH, HOG and HOF), though being essential part in previous systems, are seemingly dispensable.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{pcm2016-sent-rerank,

title = {Improving Image Captioning by Concept-based Sentence Reranking},

author = {Xirong Li and Qin Jin},

url = {http://lixirong.net/pub/pcm2016-sent-rerank.pdf},

doi = {10.1007/978-3-319-48896-7_23},

year  = {2016},

date = {2016-09-15},

booktitle = {The 17th Pacific-Rim Conference on Multimedia (PCM)},

pages = {231-240},

abstract = {This paper describes our winning entry in the ImageCLEF 2015 image sentence generation task. We improve Google’s CNN-LSTM model by introducing concept-based sentence reranking, a data-driven approach which exploits the large amounts of concept-level annotations on Flickr. Different from previous usage of concept detection that is tailored to specific image captioning models, the propose approach reranks predicted sentences in terms of their matches with detected concepts, essentially treating the underlying model as a black box. This property makes the approach applicable to a number of existing solutions. We also experiment with fine tuning on the deep language model, which improves the performance further. Scoring METEOR of 0.1875 on the ImageCLEF 2015 test set, our system outperforms the runner-up (METEOR of 0.1687) with a clear margin.},

note = {Best Paper Runner-up},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icmr2016-chisent,

title = {Adding Chinese Captions to Images},

author = { Xirong Li and Weiyu Lan and Jianfeng Dong and Hailong Liu},

url = {http://lixirong.net/pub/icmr2016_chisent.pdf},

doi = {10.1145/2911996.2912049},

year  = {2016},

date = {2016-06-09},

booktitle = {Proceedings of the 2016 ACM on International Conference on Multimedia Retrieval (ICMR)},

pages = {271--275},

abstract = {This paper extends research on automated image captioning in the dimension of language, studying how to generate Chinese sentence descriptions for unlabeled images. To evaluate image captioning in this novel context, we present Flickr8k-CN, a bilingual extension of the popular Flickr8k set. The new multimedia dataset can be used to quantitatively assess the performance of Chinese captioning and English-Chinese machine translation. The possibility of re-using existing English data and models via machine translation is investigated. Our study reveals to some extent that a computer can master two distinct languages, English and Chinese, at a similar level for describing the visual world. Data is publicly available at http://tinyurl.com/flickr8kcn},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2015-tut,

title = {Image Tag Assignment, Refinement and Retrieval},

author = {Xirong Li and Tiberio Uricchio and Lamberto Ballan and Marco Bertini and Cees G. M. Snoek and Alberto Del Bimbo},

url = {/pub/mm2015-tutorial.pdf},

doi = {10.1145/2733373.2807419},

year  = {2015},

date = {2015-10-26},

booktitle = {Proceedings of the 23rd Annual ACM Conference on Multimedia Conference (ACMMM)},

pages = {1325--1326},

abstract = {This tutorial focuses on challenges and solutions for content-based image annotation and retrieval in the context of online image sharing and tagging. We present a unified review on three closely linked problems, i.e., tag assignment, tag refinement, and tag-based image retrieval. We introduce a taxonomy to structure the growing literature, understand the ingredients of the main works, clarify their connections and difference, and recognize their merits and limitations. Moreover, we present an open-source testbed, with training sets of varying sizes and three test datasets, to evaluate methods of varied learning complexity. A selected set of eleven representative works have been implemented and evaluated. During the tutorial we provide a practice session for hands on experience with the methods, software and datasets. For repeatable experiments all data and code are online at http://www.micc.unifi.it/tagsurvey},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2015-cmrf,

title = {Image Retrieval by Cross-Media Relevance Fusion},

author = { Jianfeng Dong and Xirong Li and Shuai Liao and Jieping Xu and Duanqing Xu and Xiaoyong Du},

url = {/pub/mm2015-cmrf.pdf},

doi = {10.1145/2733373.2809929},

year  = {2015},

date = {2015-10-20},

booktitle = {ACM Multimedia},

pages = {173--176},

abstract = {How to estimate cross-media relevance between a given query and an unlabeled image is a key question in the MSR-Bing Image Retrieval Challenge. We answer the question by proposing cross-media relevance fusion, a conceptually simple framework that exploits the power of individual methods for cross-media relevance estimation. Four base cross-media relevance functions are investigated, and later combined by weights optimized on the development set. With NDCG25 of 0.5200 on the test dataset, the proposed image retrieval system secures the first place in the evaluation.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mediaeval2015-ruc,

title = {RUCMM at MediaEval 2015 Affective Impact of Movies Task: Fusion of Audio and Visual Cues},

author = {Qin Jin and Xirong Li and Haibing Cao and Yujia Huo and Shuai Liao and Gang Yang and Jieping Xu},

url = {http://ceur-ws.org/Vol-1436/Paper26.pdf},

year  = {2015},

date = {2015-09-14},

booktitle = {Working Notes Proceedings of the MediaEval 2015 Workshop},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{sigir2015-hierse,

title = {Zero-shot Image Tagging by Hierarchical Semantic Embedding},

author = {Xirong Li and Shuai Liao and Weiyu Lan and Xiaoyong Du and Gang Yang},

url = {http://lixirong.net/pub/sigir2015-hierse.pdf},

doi = {10.1145/2766462.2767773},

year  = {2015},

date = {2015-08-09},

booktitle = {SIGIR},

abstract = {Given the difficulty of acquiring labeled examples for many fine-grained visual classes, there is an increasing interest in zero-shot image tagging, aiming to tag images with novel labels that have no training examples present. Using a semantic space trained by a neural language model, the current state-of-the-art embeds both images and labels into the space, wherein cross-media similarity is computed. However, for labels of relatively low occurrence, its similarity to images and other labels can be unreliable. This paper proposes Hierarchical Semantic Embedding (HierSE), a simple model that exploits the WordNet hierarchy to improve label embedding and consequently image embedding. Moreover, we identify two good tricks, namely training the neural language model using Flickr tags instead of web documents, and using partial match instead of full match for vectorizing a WordNet node. All this lets us outperform the state-of-the-art. On a test set of over 1,500 visual object classes and 1.3 million images, the proposed model beats the current best results (18.3% versus 9.4% in hit@1).},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icassp2015-dsc-audio,

title = {Detecting semantic concepts in consumer videos using audio},

author = {Junwei Liang and Qin Jin and Xixi He and Gang Yang and Jieping Xu and Xirong Li},

url = {http://dx.doi.org/10.1109/ICASSP.2015.7178377},

doi = {10.1109/ICASSP.2015.7178377},

year  = {2015},

date = {2015-04-19},

booktitle = {International Conference on Acoustics, Speech and Signal Processing (ICASSP)},

pages = {2279--2283},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{pcm2014-tagsel,

title = {Adaptive Tag Selection for Image Annotation},

author = {Xixi He and Xirong Li and Gang Yang and Jieping Xu and Qin Jin},

url = {/pub/pcm2014-tagsel.pdf},

doi = {10.1007/978-3-319-13168-9_2},

year  = {2014},

date = {2014-12-01},

booktitle = {Proceedings of the 15th Pacific-Rim Conference on Multimedia (PCM)},

pages = {11--21},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{imageclef2014-ruc,

title = {Renmin University of China at ImageCLEF 2014 Scalable Concept Image Annotation},

author = {Xirong Li and Xixi He and Gang Yang and Qin Jin and Jieping Xu},

url = {http://ceur-ws.org/Vol-1180/CLEF2014wn-Image-LiEt2014.pdf},

year  = {2014},

date = {2014-09-15},

booktitle = {CLEF (Working Notes)},

pages = {380--385},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icmr2014-music,

title = {Source Separation Improves Music Emotion Recognition},

author = {Jieping Xu and Xirong Li and Yun Hao and Gang Yang},

url = {/pub/icmr2014-music.pdf},

doi = {10.1145/2578726.2578784},

year  = {2014},

date = {2014-04-01},

booktitle = {ACM International Conference on Multimedia Retrieval (ICMR)},

pages = {423},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icmr2014-video,

title = {Few-Example Video Event Retrieval using Tag Propagation},

author = {Masoud Mazloom and Xirong Li and Cees G. M. Snoek},

url = {/pub/icmr2014-video.pdf},

doi = {10.1145/2578726.2578793},

year  = {2014},

date = {2014-01-01},

booktitle = {ACM International Conference on Multimedia Retrieval (ICMR)},

pages = {459},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{pcm2013-crosscodebook,

title = {Cross-Codebook Image Classification},

author = {Shuai Liao and Xirong Li and Xiaoyong Du},

url = {/pub/pcm2013-crosscodebook.pdf},

doi = {10.1007/978-3-319-03731-8_46},

year  = {2013},

date = {2013-10-25},

booktitle = {The 14th Pacific-Rim Conference on Multimedia (PCM)},

pages = {497--504},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2013-tagrel.pdf,

title = {Classifying tag relevance with relevant positive and negative examples},

author = {Xirong Li and Cees G. M. Snoek},

url = {/pub/mm2013-tagrel.pdf},

doi = {10.1145/2502081.2502129},

year  = {2013},

date = {2013-10-21},

booktitle = {ACM International Conference on Multimedia (ACMMM)},

pages = {485--488},

abstract = {Image tag relevance estimation aims to automatically determine what people label about images is factually present in the pictorial content. Different from previous works, which either use only positive examples of a given tag or use positive and random negative examples, we argue the importance of relevant positive and relevant negative examples for tag relevance estimation. We propose a system that selects positive and negative examples, deemed most relevant with respect to the given tag from crowd-annotated images. While applying models for many tags could be cumbersome, our system trains efficient ensembles of Support Vector Machines per tag, enabling fast classification. Experiments on two benchmark sets show that the proposed system compares favorably against five present day methods. Given extracted visual features, for each image our system can process up to 3,787 tags per second. The new system is both effective and efficient for tag relevance estimation.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{cbmi2013-socialmedia,

title = {Evaluating Sources and Strategies for Learning Video Concepts from Social Media},

author = {Svetlana Kordumova and Xirong Li and Cees G. M. Snoek},

url = {http://lixirong.net/pub/cbmi2013-socialmedia.pdf},

doi = {10.1109/CBMI.2013.6576561},

year  = {2013},

date = {2013-06-17},

booktitle = {the 11th International Workshop on Content-Based Multimedia Indexing (CBMI)},

pages = {91--96},

abstract = {Learning video concept detectors from social media sources, such as Flickr images and YouTube videos, has the potential to address a wide variety of concept queries for video search. While the potential has been recognized by many, and progress on the topic has been impressive, we argue that two key questions, i.e., What visual tagging source is most suited for selecting positive training examples to learn video concepts? and What strategy should be used for selecting positive examples from tagged sources?, remain open. As an initial attempt to answer the two questions, we conduct an experimental study using a video search engine which is capable of learning concept detectors from social media, be it socially tagged videos or socially tagged images.Within the video search engine we investigate six strategies of positive examples selection. The performance is evaluated on the challenging TRECVID benchmark 2011 with 400 hours of Internet videos. The new experiments lead to novel and nontrivial findings: (1) tagged images are a better source for learning video concepts from the web, (2) selecting tag relevant examples as positives for learning video concepts is always beneficial and it can be done automatically and (3) the best source and strategy compare favorably against several present-day methods.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{imageclef2013-ruc,

title = {Renmin University of China at ImageCLEF 2013 Scalable Concept Image Annotation},

author = {Xirong Li and Shuai Liao and Binbin Liu and Gang Yang and Qin Jin and Jieping Xu and Xiaoyong Du},

url = {http://ceur-ws.org/Vol-1179/CLEF2013wn-ImageCLEF-LiEt2013.pdf},

year  = {2013},

date = {2013-01-01},

booktitle = {CLEF (Working Notes)},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icmr2012-geocontext,

title = {Fusing Concept Detection and Geo Context for Visual Search},

author = { Xirong Li and Cees G. M. Snoek and Marcel Worring and Arnold W. M. Smeulders},

url = {/pub/icmr2012-geocontext.pdf},

doi = {10.1145/2324796.2324801},

isbn = {978-1-4503-1329-2},

year  = {2012},

date = {2012-06-05},

booktitle = {ACM International Conference on Multimedia Retrieval (ICMR)},

pages = {4:1--4:8},

series = {ICMR '12},

abstract = {Given the proliferation of geo-tagged images, the question of how to exploit geo tags and the underlying geo context for visual search is emerging. Based on the observation that the importance of geo context varies over concepts, we propose a concept-based image search engine which fuses visual concept detection and geo context in a concept-dependent manner. Compared to individual content-based and geo-based concept detectors and their uniform combination, concept-dependent fusion shows improvements. Moreover, since the proposed search engine is trained on social-tagged images alone without the need of human interaction, it is flexible to cope with many concepts. Search experiments on 101 popular visual concepts justify the viability of the proposed solution. In particular, for 79 out of the 101 concepts, the learned weights yield improvements over the uniform weights, with a relative gain of at least 5% in terms of average precision.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2011-personalized,

title = {Personalizing automated image annotation using cross-entropy},

author = {Xirong Li and Efstratios Gavves and Cees G. M. Snoek and Marcel Worring and Arnold W. M. Smeulders},

url = {/pub/mm2011-personalized.pdf},

doi = {10.1145/2072298.2072330},

year  = {2011},

date = {2011-11-28},

booktitle = {ACM International Conference on Multimedia (ACMMM)},

pages = {233--242},

abstract = {Annotating the increasing amounts of user-contributed images in a personalized manner is in great demand. However, this demand is largely ignored by the mainstream of automated image annotation research. In this paper we aim for personalizing automated image annotation by jointly exploiting personalized tag statistics and content-based image annotation. We propose a cross-entropy based learning algorithm which personalizes a generic annotation model by learning from a user's multimedia tagging history. Using cross-entropy-minimization based Monte Carlo sampling, the proposed algorithm optimizes the personalization process in terms of a performance measurement which can be flexibly chosen. Automatic image annotation experiments with 5,315 realistic users in the social web show that the proposed method compares favorably to a generic image annotation method and a method using personalized tag statistics only. For 4,442 users the performance improves, where for 1,088 users the absolute performance gain is at least 0.05 in terms of average precision. The results show the value of the proposed method.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icmr2011-negative,

title = {Social negative bootstrapping for visual categorization},

author = {Xirong Li and Cees G. M. Snoek and Marcel Worring and Arnold W. M. Smeulders},

url = {/pub/icmr2011-negative.pdf},

doi = {10.1145/1991996.1992008},

year  = {2011},

date = {2011-04-18},

booktitle = {ACM International Conference on Multimedia Retrieval (ICMR)},

pages = {12},

abstract = {To learn classifiers for many visual categories, obtaining labeled training examples in an efficient way is crucial. Since a classifier tends to misclassify negative examples which are visually similar to positive examples, inclusion of such informative negatives should be stressed in the learning process. However, they are unlikely to be hit by random sampling, the de facto standard in literature. In this paper, we go beyond random sampling by introducing a novel social negative bootstrapping approach. Given a visual category and a few positive examples, the proposed approach adaptively and iteratively harvests informative negatives from a large amount of social-tagged images. To label negative examples without human interaction, we design an effective virtual labeling procedure based on simple tag reasoning. Virtual labeling, in combination with adaptive sampling, enables us to select the most misclassified negatives as the informative samples. Learning from the positive set and the informative negative sets results in visual classifiers with higher accuracy. Experiments on two present-day image benchmarks employing 650K virtually labeled negative examples show the viability of the proposed approach. On a popular visual categorization benchmark our precision at 20 increases by 34%, compared to baselines trained on randomly sampled negatives. We achieve more accurate visual categorization without the need of manually labeling any negatives.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{civr2010-multifeature,

title = {Unsupervised multi-feature tag relevance learning for social image retrieval},

author = {Xirong Li and Cees G. M. Snoek and Marcel Worring},

url = {/pub/civr2010-multifeature.pdf},

doi = {10.1145/1816041.1816044},

year  = {2010},

date = {2010-07-05},

booktitle = {ACM International Conference on Image and Video Retrieval (CIVR)},

pages = {10--17},

abstract = {Interpreting the relevance of a user-contributed tag with respect to the visual content of an image is an emerging problem in social image retrieval. In the literature this problem is tackled by analyzing the correlation between tags and images represented by specific visual features. Unfortunately, no single feature represents the visual content completely, e.g., global features are suitable for capturing the gist of scenes, while local features are better for depicting objects. To solve the problem of learning tag relevance given multiple features, we introduce in this paper two simple and effective methods: one is based on the classical Borda Count and the other is a method we name UniformTagger. Both methods combine the output of many tag relevance learners driven by diverse features in an unsupervised, rather than supervised, manner. Experiments on 3.5 million social-tagged images and two test sets verify our proposal. Using learned tag relevance as updated tag frequency for social image retrieval, both Borda Count and UniformTagger outperform retrieval without tag relevance learning and retrieval with single-feature tag relevance learning. Moreover, the two unsupervised methods are comparable to a state-of-the-art supervised alternative, but without the need of any training data.},

note = {best paper award},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2009-neg4free,

title = {Visual categorization with negative examples for free},

author = {Xirong Li and Cees G. M. Snoek},

url = {/pub/mm2009-neg4free.pdf},

doi = {10.1145/1631272.1631382},

year  = {2009},

date = {2009-10-19},

booktitle = {ACM International Conference on Multimedia (ACMMM)},

pages = {661--664},

abstract = {Automatic visual categorization is critically dependent on labeled examples for supervised learning. As an alternative to traditional expert labeling, social-tagged multimedia is becoming a novel yet subjective and inaccurate source of learning examples. Different from existing work focusing on collecting positive examples, we study in this paper the potential of substituting social tagging for expert labeling for creating negative examples. We present an empirical study using 6.5 million Flickr photos as a source of social tagging. Our experiments on the PASCAL VOC challenge 2008 show that with a relative loss of only 4.3% in terms of mean average precision, expert-labeled negative examples can be completely replaced by social-tagged negative examples for consumer photo categorization.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{icassp2009-userphotos,

title = {Annotating images by harnessing worldwide user-tagged photos},

author = {Xirong Li and Cees G. M. Snoek and Marcel Worring},

url = {/pub/icassp2009-userphotos.pdf},

doi = {10.1109/ICASSP.2009.4960434},

year  = {2009},

date = {2009-04-19},

booktitle = {IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP)},

pages = {3717--3720},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mir2008-tagrel,

title = {Learning tag relevance by neighbor voting for social image retrieval},

author = {Xirong Li and Cees G. M. Snoek and Marcel Worring},

url = {/pub/mir2008-tagrel.pdf},

doi = {10.1145/1460096.1460126},

year  = {2008},

date = {2008-10-30},

booktitle = {ACM International Conference on Multimedia Information Retrieval (MIR)},

pages = {180--187},

abstract = {Social image retrieval is important for exploiting the increasing amounts of amateur-tagged multimedia such as Flickr images. Since amateur tagging is known to be uncontrolled, ambiguous, and personalized, a fundamental problem is how to reliably interpret the relevance of a tag with respect to the visual content it is describing. Intuitively, if different persons label similar images using the same tags, these tags are likely to reflect objective aspects of the visual content. Starting from this intuition, we propose a novel algorithm that scalably and reliably learns tag relevance by accumulating votes from visually similar neighbors. Further, treated as tag frequency, learned tag relevance is seamlessly embedded into current tag-based social image retrieval paradigms.



Preliminary experiments on one million Flickr images demonstrate the potential of the proposed algorithm. Overall comparisons for both single-word queries and multiple-word queries show substantial improvement over the baseline by learning and using tag relevance. Specifically, compared with the baseline using the original tags, on average, retrieval using improved tags increases mean average precision by 24%, from 0.54 to 0.67. Moreover, simulated experiments indicate that performance can be improved further by scaling up the amount of images used in the proposed neighbor voting algorithm.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2007-query-concept-mapping,

title = {The importance of query-concept-mapping for automatic video retrieval},

author = {Dong Wang and Xirong Li and Jianmin Li and Bo Zhang},

url = {/pub/mm2007-query-concept-mapping.pdf},

doi = {10.1145/1291233.1291293},

year  = {2007},

date = {2007-09-24},

booktitle = {ACM International Conference on Multimedia (ACMMM)},

pages = {285--288},

crossref = {DBLP:conf/mm/2007},

abstract = {A new video retrieval paradigm of query-by-concept emerges recently. However, it remains unclear how to exploit the detected concepts in retrieval given a multimedia query. In this paper, we point out that it is important to map the query to a few relevant concepts instead of search with all concepts. In addition, we show that solving this problem through both text and image inputs are effective for search, and it is possible to determine the number of related concepts by a language modeling approach. Experimental evidence is obtained on the automatic search task of TRECVID 2006 using a large lexicon of 311 learned semantic concept detectors.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{civr2007-conceptsubspace,

title = {Video search in concept subspace: a text-like paradigm},

author = {Xirong Li and Dong Wang and Jianmin Li and Bo Zhang},

url = {/pub/civr2007-conceptsubspace.pdf},

doi = {10.1145/1282280.1282366},

year  = {2007},

date = {2007-07-09},

booktitle = {ACM International Conference on Image and Video Retrieval (CIVR)},

pages = {603--610},

abstract = {Though both quantity and quality of semantic concept detection in video are continuously improving, it still remains unclear how to exploit these detected concepts as semantic indices in video search, given a specific query. In this paper, we tackle this problem and propose a video search framework which operates like searching text documents. Noteworthy for its adoption of the well-founded text search principles, this framework first selects a few related concepts for a given query, by employing a tf-idf like scheme, called c-tf-idf, to measure the informativeness of the concepts to this query. These selected concepts form a concept subspace. Then search can be conducted in this concept subspace, either by a Vector Model or a Language Model. Further, two algorithms, i.e., Linear Summation and Random Walk through Concept-Link, are explored to combine the concept search results and other baseline search results in a reranking scheme. This framework is both effective and efficient. Using a lexicon of 311 concepts from the LSCOM concept ontology, experiments conducted on the TRECVID 2006 search data set show that: when used solely, search within the concept subspace achieves the state-of-the-art concept search result; when used to rerank the baseline results, it can improve over the top 20 automatic search runs in TRECVID 2006 on average by approx. 20%, on the most significant one by approx. 50%, all within 180 milliseconds on a normal PC.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{mm2006-imageanno,

title = {Image annotation by large-scale content-based image retrieval},

author = {Xirong Li and Le Chen and Lei Zhang and Fuzong Lin and Wei-Ying Ma},

url = {/pub/mm2006-imageanno.pdf},

doi = {10.1145/1180639.1180764},

year  = {2006},

date = {2006-10-23},

booktitle = {ACM International Conference on Multimedia (ACMMM)},

pages = {607--610},

abstract = {Image annotation has been an active research topic in recent years due to its potentially large impact on both image understanding and Web image search. In this paper, we target at solving the automatic image annotation problem in a novel search and mining framework. Given an uncaptioned image, first in the search stage, we perform content-based image retrieval (CBIR) facilitated by high-dimensional indexing to find a set of visually similar images from a large-scale image database. The database consists of images crawled from the World Wide Web with rich annotations, e.g. titles and surrounding text. Then in the mining stage, a search result clustering technique is utilized to find most representative keywords from the annotations of the retrieved image subset. These keywords, after salience ranking, are finally used to annotate the uncaptioned image. Based on search technologies, this framework does not impose an explicit training stage, but efficiently leverages large-scale and well-annotated images, and is potentially capable of dealing with unlimited vocabulary. Based on 2.4 million real Web images, comprehensive evaluation of image annotation on Corel and U. Washington image databases show the effectiveness and efficiency of the proposed approach.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}