【科研】科研神器

>       1. ├──  config

>  
>       2. │    └── defaults.py  - here's the default config file.
>  
>       3. │
>  
>       4. │
>  
>       5. ├──  configs  
>  
>       6. │    └── train_mnist_softmax.yml  - here's the specific config file for specific model or dataset.
>  
>       7. │
>  
>       8. │
>  
>       9. ├──  data  
>  
>       10. │    └── datasets  - here's the datasets folder that is responsible for all data handling.
>  
>       11. │    └── transforms  - here's the data preprocess folder that is responsible for all data augmentation.
>  
>       12. │    └── build.py       - here's the file to make dataloader.
>  
>       13. │    └── collate_batch.py   - here's the file that is responsible for merges a list of samples to form a mini-batch.
>  
>       14. │
>  
>       15. │
>  
>       16. ├──  engine
>  
>       17. │   ├── trainer.py     - this file contains the train loops.
>  
>       18. │   └── inference.py   - this file contains the inference process.
>  
>       19. │
>  
>       20. │
>  
>       21. ├── layers              - this folder contains any customed layers of your project.
>  
>       22. │   └── conv_layer.py
>  
>       23. │
>  
>       24. │
>  
>       25. ├── modeling            - this folder contains any model of your project.
>  
>       26. │   └── example_model.py
>  
>       27. │
>  
>       28. │
>  
>       29. ├── solver             - this folder contains optimizer of your project.
>  
>       30. │   └── build.py
>  
>       31. │   └── lr_scheduler.py
>  
>       32. │  
>  
>       33. │
>  
>       34. ├──  tools                - here's the train/test model of your project.
>  
>       35. │    └── train_net.py  - here's an example of train model that is responsible for the whole pipeline.
>  
>       36. │
>  
>       37. │
>  
>       38. └── utils
>  
>       39. │    ├── logger.py
>  
>       40. │    └── any_other_utils_you_need
>  
>       41. │
>  
>       42. │
>  
>       43. └── tests     - this foler contains unit test of your project.
>  
>       44.      ├── test_data_sampler.py
>  
>  
>  
>  
>     AI助手

 parser.add_argument('--batch_size', type=int, default=2048,

    
                 help='batch size')  # 8192
    
 parser.add_argument('--save_dir', type=str,
    
                 help="save exp floder name", default="exp1_sandwich")
    
    
    
    
    AI助手

 parser.add_argument('--finetune', action='store_true',

    
                 help='finetune model with distill')
    
    
    
    
    AI助手

    pip install pyyaml
    
    AI助手

 name: tosan

    
 age: 22
    
 skill:
    
   name1: coding
    
   time: 2years
    
 job:
    
   - name2: JD
    
     pay: 2k
    
   - name3: HW
    
     pay: 4k
    
    
    
    
    AI助手

 import yaml

    
  
    
 f = open("configs/test.yml", "r")
    
  
    
 y = yaml.load(f)
    
  
    
 print(y)
    
    
    
    
    AI助手

 YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.

    
   y = yaml.load(f)
    
 {'name': 'tosan', 'age': 22, 'skill': {'name1': 'coding', 'time': '2years'}, 'job': [{'name2': 'JD', 'pay': '2k'}, {'name3': 'HW', 'pay': '4k'}]}
    
    
    
    
    AI助手

 import yaml

    
  
    
 f = open("configs/test.yml", "r")
    
  
    
 y = yaml.load(f, Loader=yaml.FullLoader)
    
  
    
 print(y)
    
    
    
    
    AI助手

 content_dict = {

    
  'name':"ch",
    
 }
    
  
    
 f = open("./config.yml","w")
    
  
    
 print(yaml.dump(content_dict, f))
    
    
    
    
    AI助手

 # 支持数字，整形、float

    
 pi: 3.14 
    
  
    
 # 支持布尔变量
    
 islist: true
    
 isdict: false
    
  
    
 # 支持None 
    
 cash: ~
    
  
    
 # 时间日期采用ISO8601
    
 time1: 2021-6-9 21:59:43.10-05:00
    
  
    
 #强制转化类型
    
 int_to_str: !!str 123
    
 bool_to_str: !!str true
    
  
    
 # 支持list
    
 - 1
    
 - 2
    
 - 3
    
  
    
 # 复合list和dict
    
 test2:
    
   - name: xxx
    
     attr1: sunny
    
     attr2: rainy
    
     attr3: cloudy
    
    
    
    
    AI助手

 import logging

    
  
    
 logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
    
  
    
 logging.info("program start")
    
    
    
    
    AI助手

 import logging

    
  #先声明一个 Logger 对象
    
 logger = logging.getLogger(__name__)
    
 logger.setLevel(level=logging.INFO)
    
 #然后指定其对应的 Handler 为 FileHandler 对象
    
 handler = logging.FileHandler('Alibaba.log')
    
 #然后 Handler 对象单独指定了 Formatter 对象单独配置输出格式
    
 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    
 handler.setFormatter(formatter)
    
 logger.addHandler(handler)
    
    
    
    
    AI助手

 # process argparse & yaml

    
 if  args.config:
    
     opt = vars(args)
    
     args = yaml.load(open(args.config), Loader=yaml.FullLoader)
    
     opt.update(args)
    
     args = opt
    
 else:  # yaml priority is higher than args
    
     opt = yaml.load(open(args.config), Loader=yaml.FullLoader)
    
     opt.update(vars(args))
    
     args = argparse.Namespace(**opt)
    
    
    
    
    AI助手

 exp

    
  - 实验名+日期
    
   - runs: tensorboard保存的文件
    
   - weights: 权重文件
    
   - config.yml: 配置文件
    
   - scripts: 核心文件备份
    
    - train.py
    
    - xxxxx.py
    
    
    
    
    AI助手

 import logging

    
 import argparse
    
 import yaml 
    
  
    
 parser = argparse.ArgumentParser("ResNet20-cifar100")
    
 parser.add_argument('--batch_size', type=int, default=2048,
    
                 help='batch size')  # 8192
    
 parser.add_argument('--learning_rate', type=float,
    
                 default=0.1, help='init learning rate')  parser.add_argument('--config', help="configuration file",
    
                 type=str, default="configs/meta.yml")
    
 parser.add_argument('--save_dir', type=str,
    
                 help="save exp floder name", default="exp1")
    
 args = parser.parse_args()
    
  
    
 # process argparse & yaml
    
 if  args.config:
    
     opt = vars(args)
    
     args = yaml.load(open(args.config), Loader=yaml.FullLoader)
    
     opt.update(args)
    
     args = opt
    
 else:  # yaml priority is higher than args
    
     opt = yaml.load(open(args.config), Loader=yaml.FullLoader)
    
     opt.update(vars(args))
    
     args = argparse.Namespace(**opt)
    
  
    
 args.exp_name = args.save_dir + "_" + datetime.datetime.now().strftime("%mM_%dD_%HH") + "_" + \
    
     "{:04d}".format(random.randint(0, 1000))
    
  
    
 # 文件处理
    
 if not os.path.exists(os.path.join("exp", args.exp_name)):
    
     os.makedirs(os.path.join("exp", args.exp_name))
    
  
    
  
    
 # 日志文件
    
 log_format = "%(asctime)s %(message)s"
    
 logging.basicConfig(stream=sys.stdout, level=logging.INFO,
    
                 format=log_format, datefmt="%m/%d %I:%M:%S %p")
    
  
    
 fh = logging.FileHandler(os.path.join("exp", args.exp_name, 'log.txt'))
    
 fh.setFormatter(logging.Formatter(log_format))
    
 logging.getLogger().addHandler(fh)
    
 logging.info(args)
    
  
    
 # 配置文件
    
 with open(os.path.join("exp", args.exp_name, "config.yml"), "w") as f:
    
     yaml.dump(args, f)
    
  
    
 # Tensorboard文件
    
 writer = SummaryWriter("exp/%s/runs/%s-%05d" %
    
                    (args.exp_name, time.strftime("%m-%d", time.localtime()), random.randint(0, 100)))
    
  
    
 # 文件备份
    
 create_exp_dir(os.path.join("exp", args.exp_name),
    
            scripts_to_save=glob.glob('*.py'))
    
  
    
 def create_exp_dir(path, scripts_to_save=None):
    
     if not os.path.exists(path):
    
     os.mkdir(path)
    
     print('Experiment dir : {}'.format(path))
    
  
    
     if scripts_to_save is not None:
    
     if not os.path.exists(os.path.join(path, 'scripts')):
    
         os.mkdir(os.path.join(path, 'scripts'))
    
     for script in scripts_to_save:
    
         dst_file = os.path.join(path, 'scripts', os.path.basename(script))
    
         shutil.copyfile(script, dst_file)
    
    
    
    
    AI助手