AutoML | Literature on Neural Architecture Search

In: Chaudhuri, Kamalika; Jegelka, Stefanie; Song, Le; Szepesvari, Csaba; Niu, Gang; Sabato, Sivan (Ed.): Proceedings of the 39th International Conference on Machine Learning, pp. 20810–20826, PMLR, 2022.

Abstract | Links | BibTeX

@inproceedings{pmlr-v162-sun22c,

title = {MAE-DET: Revisiting Maximum Entropy Principle in Zero-Shot NAS for Efficient Object Detection},

author = {Zhenhong Sun and Ming Lin and Xiuyu Sun and Zhiyu Tan and Hao Li and Rong Jin},

editor = {Kamalika Chaudhuri and Stefanie Jegelka and Le Song and Csaba Szepesvari and Gang Niu and Sivan Sabato},

url = {https://proceedings.mlr.press/v162/sun22c.html},

year  = {2022},

date = {2022-07-01},

urldate = {2022-07-01},

booktitle = {Proceedings of the 39th International Conference on Machine Learning},

volume = {162},

pages = {20810--20826},

publisher = {PMLR},

series = {Proceedings of Machine Learning Research},

abstract = {In object detection, the detection backbone consumes more than half of the overall inference cost. Recent researches attempt to reduce this cost by optimizing the backbone architecture with the help of Neural Architecture Search (NAS). However, existing NAS methods for object detection require hundreds to thousands of GPU hours of searching, making them impractical in fast-paced research and development. In this work, we propose a novel zero-shot NAS method to address this issue. The proposed method, named MAE-DET, automatically designs efficient detection backbones via the Maximum Entropy Principle without training network parameters, reducing the architecture design cost to nearly zero yet delivering the state-of-the-art (SOTA) performance. Under the hood, MAE-DET maximizes the differential entropy of detection backbones, leading to a better feature extractor for object detection under the same computational budgets. After merely one GPU day of fully automatic design, MAE-DET innovates SOTA detection backbones on multiple detection benchmark datasets with little human intervention. Comparing to ResNet-50 backbone, MAE-DET is $+2.0%$ better in mAP when using the same amount of FLOPs/parameters, and is $1.54$ times faster on NVIDIA V100 at the same mAP. Code and pre-trained models are available here (https://github.com/alibaba/lightweight-neural-architecture-search).},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

Close

2273.

Sun, Zhenhong; Lin, Ming; Sun, Xiuyu; Tan, Zhiyu; Li, Hao; Jin, Rong

MAE-DET: Revisiting Maximum Entropy Principle in Zero-Shot NAS for Efficient Object Detection Proceedings Article

In: Chaudhuri, Kamalika; Jegelka, Stefanie; Song, Le; Szepesvari, Csaba; Niu, Gang; Sabato, Sivan (Ed.): Proceedings of the 39th International Conference on Machine Learning, pp. 20810–20826, PMLR, 2022.

Abstract | Links | BibTeX

@inproceedings{pmlr-v162-sun22cb,

title = {MAE-DET: Revisiting Maximum Entropy Principle in Zero-Shot NAS for Efficient Object Detection},

author = {Zhenhong Sun and Ming Lin and Xiuyu Sun and Zhiyu Tan and Hao Li and Rong Jin},

editor = {Kamalika Chaudhuri and Stefanie Jegelka and Le Song and Csaba Szepesvari and Gang Niu and Sivan Sabato},

url = {https://proceedings.mlr.press/v162/sun22c.html},

year  = {2022},

date = {2022-07-01},

urldate = {2022-07-01},

booktitle = {Proceedings of the 39th International Conference on Machine Learning},

volume = {162},

pages = {20810--20826},

publisher = {PMLR},

series = {Proceedings of Machine Learning Research},

abstract = {In object detection, the detection backbone consumes more than half of the overall inference cost. Recent researches attempt to reduce this cost by optimizing the backbone architecture with the help of Neural Architecture Search (NAS). However, existing NAS methods for object detection require hundreds to thousands of GPU hours of searching, making them impractical in fast-paced research and development. In this work, we propose a novel zero-shot NAS method to address this issue. The proposed method, named MAE-DET, automatically designs efficient detection backbones via the Maximum Entropy Principle without training network parameters, reducing the architecture design cost to nearly zero yet delivering the state-of-the-art (SOTA) performance. Under the hood, MAE-DET maximizes the differential entropy of detection backbones, leading to a better feature extractor for object detection under the same computational budgets. After merely one GPU day of fully automatic design, MAE-DET innovates SOTA detection backbones on multiple detection benchmark datasets with little human intervention. Comparing to ResNet-50 backbone, MAE-DET is $+2.0%$ better in mAP when using the same amount of FLOPs/parameters, and is $1.54$ times faster on NVIDIA V100 at the same mAP. Code and pre-trained models are available here (https://github.com/alibaba/lightweight-neural-architecture-search).},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

Close

2272.

Marchisio, Alberto; Mrazek, Vojtech; Massa, Andrea; Bussolino, Beatrice; Martina, Maurizio; Shafique, Muhammad

HARNAS: Neural Architecture Search Jointly Optimizing for Hardware Efficiency and Adversarial Robustness of Convolutional and Capsule Networks Proceedings Article

In: DyNN workshop at the 39th International Conference on Machine Learning, 2022.

Links | BibTeX

2271.

Malkova, Aleksandra

Reconstruction of Radio Power Maps with Deep Neural Networks in a Weakly Labeled Learning Context PhD Thesis

Universite Grenoble Alpes, 2022.

Links | BibTeX

2270.

Zhou, J.; Chen, F.; Lu, G.

Pruning Based Training-Free Neural Architecture Search Proceedings Article

In: 2022 IEEE International Conference on Multimedia and Expo (ICME), pp. 1-6, IEEE Computer Society, Los Alamitos, CA, USA, 2022.

Abstract | Links | BibTeX

2269.

Poliakov, E.; Luu, V.; Tran, V.; Huang, C.

Model Compression via Structural Pruning and Feature Distillation for Accurate Multi-Spectral Object Detection on Edge-Devices Proceedings Article

In: 2022 IEEE International Conference on Multimedia and Expo (ICME), pp. 1-6, IEEE Computer Society, Los Alamitos, CA, USA, 2022.

Abstract | Links | BibTeX

2268.

Liao, W.; Zhang, Z.; Wang, X.; Yin, T.; Liu, H.; Ren, Z.; Yin, J.; Feng, S.

Distilldarts: Network Distillation for Smoothing Gradient Distributions in Differentiable Architecture Search Proceedings Article

In: 2022 IEEE International Conference on Multimedia and Expo (ICME), pp. 1-6, IEEE Computer Society, Los Alamitos, CA, USA, 2022.

Abstract | Links | BibTeX

2267.

Chu, Jianjun; Yu, Xiaoshan; Yang, Shangshang; Qiu, Jianfeng; Wang, Qijun

Architecture entropy sampling-based evolutionary neural architecture search and its application in osteoporosis diagnosis Journal Article

In: Complex & Intelligent Systems , 2022.

Links | BibTeX

2266.

Sapra, Dolly

Efficient neural architectures for edge devices Supervisors PhD Thesis

2022.

Links | BibTeX

2265.

Pinos, Michal; Mrazek, Vojtech; Sekanina, Lukas

Evolutionary approximation and neural architecture search Journal Article

In: Genet Program Evolvable Mach (2022), 2022.

Links | BibTeX

2264.

Rosero, Santiago Vladimir Gomez

Evolving Deep Neural Network Architectures for Time Series Data PhD Thesis

2022.

Links | BibTeX

2263.

Lima, Ricardo H. R.; Magalhães, Dimmy; Pozo, Aurora; Mendiburu, Alexander; Santana, Roberto

A grammar-based GP approach applied to the design of deep neural networks Journal Article

In: Genetic Programming and Evolvable Machines , 2022.

Links | BibTeX

2262.

Hendrickx, Lotte; Ranst, Wiebe Van; Goedemé, Toon

Hot-Started NAS for Task-Specific Embedded Applications Proceedings Article

In: Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) Workshops, pp. 1971-1978, 2022.

Links | BibTeX

2261.

Pinos, Michal; Mrazek, Vojtech; Sekanina, Lukas

Evolutionary approximation and neural architecture search Journal Article

In: Genetic Programming and Evolvable Machines, 2022.

Links | BibTeX

2260.

YANG, Yibo; SHEN, Zhengyang; LI, Huan; LIN, Zhouchen

Optimization-Inspired Manual Architecture Design and Neural Architecture Search Journal Article

In: SCIENCE CHINA Information Sciences, 2022.

BibTeX

2259.

Yue, Zhixiong; Guo, Pengxin; Zhang, Yu; Liang, Jie

Learning Feature Alignment Architecture for Domain Adaptation Proceedings Article

In: IJCNN 2022, 2022.

Links | BibTeX

2258.

Ayman, Afiya; Martinez, Juan; Pugliese, Philip; Dubey, Abhishek; Laszka, Aron

Neural Architecture and Feature Search for Predicting the Ridership of Public Transportation Routes Proceedings Article

In: proceedings of the 8th IEEE International Conference on Smart Computing (SMARTCOMP 2022), 2022.

Links | BibTeX

2257.

He, Zhimin; Su, Junjian; Chen, Chuangtao; Pan, Minghua; Situ, Haozhen

Search space pruning for quantum architecture search Proceedings Article

In: The European Physical Journal Plus , 2022.

Links | BibTeX

2256.

Zhang, Xinbang; Jin, Qizhao; Yu, Tingzhao; Xiang, Shiming; Kuang, Qiuming; Prinet, Véronique; Pan, Chunhong

Multi-modal spatio-temporal meteorological forecasting with deep neural network Proceedings Article

In: ISPRS journal of photogrammetry and remote sensing, 2022.

Links | BibTeX

2255.

Zhang, Xinbang; Jin, Qizhao; Yu, Tingzhao; Xiang, Shiming; Kuang, Qiuming; Prinet, Véronique; Pan, Chunhong

Multi-modal spatio-temporal meteorological forecasting with deep neural network Proceedings Article

In: ISPRS journal of photogrammetry and remote sensing, 2022.

Links | BibTeX

2254.

Zhang, Xinbang; Jin, Qizhao; Yu, Tingzhao; Xiang, Shiming; Kuang, Qiuming; Prinet, Véronique; Pan, Chunhong

Multi-modal spatio-temporal meteorological forecasting with deep neural network Proceedings Article

In: ISPRS journal of photogrammetry and remote sensing, 2022.

Links | BibTeX

2253.

Li, Xiao; Lei, Lin; Zhang, Caiguang; Kuang, Gangyao

Multimodal Semantic Consistency-Based Fusion Architecture Search for Land Cover Classification Technical Report

2022.

Links | BibTeX

2252.

Kishore, Jaydeep; Mukherjee, Snehasis

Autotuned Classication Based on Knowledge Transferred from Self-supervised Models Technical Report

2022.

Links | BibTeX

2251.

Yan, Chengrun

Automated Machine Learning Under Resource Constraints PhD Thesis

2022.

Links | BibTeX

2250.

Louati, Hassen; Bechikh, Slim; Louati, Ali; Aldaej, Abdulaziz; Said, Lamjed Ben

Joint design and compression of convolutional neural networks as a Bi-level optimization problem Journal Article

In: Neural Computing and Applications , 2022.

Links | BibTeX

2249.

Luo, Jiapeng; Wang, Zhongfeng

Automatically search an optimal face detector for a specific deployment environment Journal Article

In: EURASIP Journal on Advances in Signal Processing, 2022.

Links | BibTeX

2248.

Kung, S. Y.

XNAS: A Regressive/Progressive NAS for Deep Learning Journal Article

In: ACM Trans. Sen. Netw., 2022, ISSN: 1550-4859, (Just Accepted).

Abstract | Links | BibTeX

@article{10.1145/3543669,

title = {XNAS: A Regressive/Progressive NAS for Deep Learning},

author = {S. Y. Kung},

url = {https://doi.org/10.1145/3543669},

doi = {10.1145/3543669},

issn = {1550-4859},

year  = {2022},

date = {2022-05-01},

urldate = {2022-05-01},

journal = {ACM Trans. Sen. Netw.},

publisher = {Association for Computing Machinery},

address = {New York, NY, USA},

abstract = {Deep learning has achieved great and broad breakthroughs in many real-world applications. In particular, the task of training the network parameters has been masterly handled by the Back-Propagation (BP) learning. However, the pursuit on optimal network structures remains largely an art of trial-and-error. This prompts some urgency to explore an architecture engineering process, collectively known as “Neural Architecture Search” (NAS). In general, NAS is a design software system for automating the search of effective neural architecture. This paper proposes a X-learning NAS (XNAS) to automatically train network’s structure and parameters. Our theoretical footing is built upon the subspace and correlation analyses between the input layer, hidden layer, and output layer. The design strategy hinges upon the underlying principle that the network should be coerced to learn how to structurally improve the input/output correlation successively (i.e. layer by layer). It embraces both PNAS and RNAS, short for Progressive and Regressive NAS respectively. For unsupervised RNAS, PCA (Principal Component Analysis) is a classic tool for subspace analyses. By further incorporating teacher’s guidance, PCA can be extended to Regression Component Analysis (RCA) to facilitate supervised NAS design. This allows the machine to extract components most critical to the targeted learning objective. We shall further extend the subspace analysis from MLPs to CNNs, via introduction of Convolutional-PCA (CPCA) or, more simply, Deep-PCA (DPCA). The supervised variant of DPCA will be named Deep-RCA (DRCA). The subspace analyses allow us to compute optimal eigenvectors (resp. eigen-filters) and principal components (resp. eigen-channels) for optimal NAS design of MLPs (resp. CNNs). Based on the theoretical analysis, a X-Learning paradigm is developed to jointly learn the structure and parameters of learning models. The objective is to reduce the network complexity while retaining (and sometimes improving) the performance. With carefully pre-selected baseline models, X-Learning has shown great successes in numerous classification-type and/or regression-type applications. We have applied X-learning to the ImageNet datasets for classification and div2k for image enhancements By applying X-Learning to two types of baseline models, MobileNet and ResNet, both the low-power and high-performance application categories can be supported. Our X-learning experiments include the ImageNet datasets for classification and div2k for image enhancements. Our simulations confirm that X-learning is by and large very competitive relative to the state-of-the-arts approaches.},

note = {Just Accepted},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

Close

Deep learning has achieved great and broad breakthroughs in many real-world applications. In particular, the task of training the network parameters has been masterly handled by the Back-Propagation (BP) learning. However, the pursuit on optimal network structures remains largely an art of trial-and-error. This prompts some urgency to explore an architecture engineering process, collectively known as “Neural Architecture Search” (NAS). In general, NAS is a design software system for automating the search of effective neural architecture. This paper proposes a X-learning NAS (XNAS) to automatically train network’s structure and parameters. Our theoretical footing is built upon the subspace and correlation analyses between the input layer, hidden layer, and output layer. The design strategy hinges upon the underlying principle that the network should be coerced to learn how to structurally improve the input/output correlation successively (i.e. layer by layer). It embraces both PNAS and RNAS, short for Progressive and Regressive NAS respectively. For unsupervised RNAS, PCA (Principal Component Analysis) is a classic tool for subspace analyses. By further incorporating teacher’s guidance, PCA can be extended to Regression Component Analysis (RCA) to facilitate supervised NAS design. This allows the machine to extract components most critical to the targeted learning objective. We shall further extend the subspace analysis from MLPs to CNNs, via introduction of Convolutional-PCA (CPCA) or, more simply, Deep-PCA (DPCA). The supervised variant of DPCA will be named Deep-RCA (DRCA). The subspace analyses allow us to compute optimal eigenvectors (resp. eigen-filters) and principal components (resp. eigen-channels) for optimal NAS design of MLPs (resp. CNNs). Based on the theoretical analysis, a X-Learning paradigm is developed to jointly learn the structure and parameters of learning models. The objective is to reduce the network complexity while retaining (and sometimes improving) the performance. With carefully pre-selected baseline models, X-Learning has shown great successes in numerous classification-type and/or regression-type applications. We have applied X-learning to the ImageNet datasets for classification and div2k for image enhancements By applying X-Learning to two types of baseline models, MobileNet and ResNet, both the low-power and high-performance application categories can be supported. Our X-learning experiments include the ImageNet datasets for classification and div2k for image enhancements. Our simulations confirm that X-learning is by and large very competitive relative to the state-of-the-arts approaches.

Close

2247.

AL-Ghamdi, Abdullah S. AL-Malaise; Ragab, Mahmoud; AlGhamdi, Saad Abdulla; Asseri, Amer H.; ans Deepika Koundal, Romany F. Mansour

Cognitive Computing Paradigms for Medical Big Data Processing and its Trends Journal Article

In: Computational Intelligence and Neuroscience, 2022.

Links | BibTeX

2246.

Xiao, Anqi; Shen, Biluo; Shi, Xiaojing; Zhang, Zhe; Zhang, Zeyu; Tian, Jie; Ji, Nan; Hu, Zhenhua

Intraoperative Glioma Grading Using Neural Architecture Search and Multi-modal Imaging Journal Article

In: IEEE Engineering in Medicine and Biology Society , 2022.

Links | BibTeX

2245.

Spurlock, Kyle; Elgazzar, Heba

A genetic mixed-integer optimization of neural network hyper-parameters Journal Article

In: The Journal of Supercomputing, 2022.

Links | BibTeX

2244.

N., Sasikaladevi; A., Revathi

Neural Architecture Search Network for the Diagnosis of COVID From the Radiographic Images Journal Article

In: Applications of Computational Science in Artificial Intelligence, 2022.

Links | BibTeX

2243.

Chen, Luoyu

A Progressive and Joint Method for Micro and Macro Architecture Search Journal Article

In: Journal of Physics: Conference Series, vol. 2253, no. 1, pp. 012019, 2022.

Abstract | Links | BibTeX

2242.

Yang, Ying; Zhang, Xu; Pan, Hu

Path-Based Computation Encoder for Neural Architecture Search Journal Article

In: Journal of Information Processing Systems , vol. 18, 2022.

Links | BibTeX

2241.

Du, Yuxuan; Huang, Tao; You, Shan; Hsieh, Min-Hsiu; Tao, Dacheng

Quantum circuit architecture search for variational quantum algorithms Journal Article

In: npj Quantum Information volume, 2022.

Links | BibTeX

2240.

Meng, Hang; Li, Fei; Yang, Zhen; Zhu, Qiang; Fan, Chaogang; Zhan, Shu

Salient Patch Based NAS for Grading of Colorectal Cancer Histology Images Proceedings Article

In: The 13th International Multi-Conference on Complexity, Informatics and Cybernetics: IMCIC 2022, 2022.

Links | BibTeX

2239.

Wang, Yong; Qiao, Xiaobin; Wang, Gai-Ge

Architecture evolution of convolutional neural network using monarch butterfly optimization Journal Article

In: Journal of Ambient Intelligence and Humanized Computing , 2022.

Links | BibTeX

2238.

Cai, Han; Lin, Ji; Lin, Yujun; Liu, Zhijian; Tang, Haotian; Wang, Hanrui; Zhu, Ligeng; Han, Song

Enable Deep Learning on Mobile Devices: Methods, Systems, and Applications Journal Article

In: ACM Trans. Des. Autom. Electron. Syst., vol. 27, no. 3, 2022, ISSN: 1084-4309.

Abstract | Links | BibTeX

@article{10.1145/3486618,

title = {Enable Deep Learning on Mobile Devices: Methods, Systems, and Applications},

author = {Han Cai and Ji Lin and Yujun Lin and Zhijian Liu and Haotian Tang and Hanrui Wang and Ligeng Zhu and Song Han},

url = {https://doi.org/10.1145/3486618},

doi = {10.1145/3486618},

issn = {1084-4309},

year  = {2022},

date = {2022-03-01},

urldate = {2022-03-01},

journal = {ACM Trans. Des. Autom. Electron. Syst.},

volume = {27},

number = {3},

publisher = {Association for Computing Machinery},

address = {New York, NY, USA},

abstract = {Deep neural networks (DNNs) have achieved unprecedented success in the field of artificial intelligence (AI), including computer vision, natural language processing, and speech recognition. However, their superior performance comes at the considerable cost of computational complexity, which greatly hinders their applications in many resource-constrained devices, such as mobile phones and Internet of Things (IoT) devices. Therefore, methods and techniques that are able to lift the efficiency bottleneck while preserving the high accuracy of DNNs are in great demand to enable numerous edge AI applications. This article provides an overview of efficient deep learning methods, systems, and applications. We start from introducing popular model compression methods, including pruning, factorization, quantization, as well as compact model design. To reduce the large design cost of these manual solutions, we discuss the AutoML framework for each of them, such as neural architecture search (NAS) and automated pruning and quantization. We then cover efficient on-device training to enable user customization based on the local data on mobile devices. Apart from general acceleration techniques, we also showcase several task-specific accelerations for point cloud, video, and natural language processing by exploiting their spatial sparsity and temporal/token redundancy. Finally, to support all these algorithmic advancements, we introduce the efficient deep learning system design from both software and hardware perspectives.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}