eval log

eval log

eval log

We need to define each sequence as an independent dataset in mmsegmentation.

• tools/{tent,test,cotta}.py

  # select sequence
      with open(seq_info_path, 'r') as file:
          reader = csv.reader(file)
          next(reader) # skip header row
          for row in reader: 
              if True:# condition that filters seq. e.g. row[0]=='.....'
                  seq_id_list.append(row[0])
  
  # define config for each sequence
      seq_cfg_list =[]
      for i, seq in enumerate(os.listdir(os.path.join(cfg.data.test.data_root,cfg.data.test.img_dir))):
          if seq in seq_id_list:
              globals()["cfg.data.test{}".format(i)] = deepcopy(cfg.data.test)
              globals()["cfg.data.test{}".format(i)].img_dir = os.path.join(cfg.data.test.img_dir,seq)
              globals()["cfg.data.test{}".format(i)].ann_dir = os.path.join(cfg.data.test.ann_dir,seq)
              seq_cfg_list.append(globals()["cfg.data.test{}".format(i)])
  
  # build dataset and dataloader
      datasets = [build_dataset(seq) for seq in seq_cfg_list]#, build_dataset(cfg.data.test1), build_dataset(cfg.data.test2),build_dataset(cfg.data.test3)]
      data_loaders = [build_dataloader(
          dataset,
          samples_per_gpu=1,
          workers_per_gpu=cfg.data.workers_per_gpu,
          dist=distributed,
          shuffle=False) for dataset in datasets]
  

• mmseg/datasets/shift.py

  img_infos = []
  if split is not None:
      with open(split) as f:
          for line in f:
              img_name = line.strip()
              img_info = dict(filename=img_name + img_suffix)
              if ann_dir is not None:
                  seg_map = img_name + seg_map_suffix
                  img_info['ann'] = dict(seg_map=seg_map)
              img_infos.append(img_info)
  else:
      for img in mmcv.scandir(img_dir, img_suffix, recursive=True):
          img_info = dict(filename=img)
          if ann_dir is not None:
              seg_map = img.replace(img_suffix, seg_map_suffix)
              img_info['ann'] = dict(seg_map=seg_map)
          img_infos.append(img_info)
  print_log(f'Loaded {len(img_infos)} images', logger=get_root_logger())
  # note that sort the squence to conduct contintual test-time adaptation
  img_infos_sorted = sorted(img_infos, key=lambda x: x['filename'])[:2]
  return img_infos_sorted 
  

1. Test each sequence independently to obtain mIoU and mAcc on each category.
2. Calculate the average of all sequences for each category.
3. Calculate the overall average.

• tools/{tent,test,cotta}.py after dataset.evaluate on each sequence, we store the results into a json file. Finally, res_process() aggregate all the results of each sequence and outoput the last evaluation results.

          if args.eval:
              _, eval_res,_ = dataset.evaluate(outputs, args.eval, **kwargs)
              out_dir = './Test_on_{}/tent_eval/'.format(cfg.data_split_type)
              if not os.path.exists(out_dir):
                  os.makedirs(out_dir + 'res')
              mmcv.dump(eval_res, out_dir + 'res/{}.json'.format(seq_name), indent=4)
  res_process(out_dir,cfg.csv_root)
  

Note that evaluation for each sequence requires reloading pretrained parameters of source model.

• tools/res_process.py

  def res_process(res_path,csv_root,in_domain=False):
          # Initialize the dictionary
      json_dict = {}
      seq_path = os.path.join(res_path,'res') 
      # Loop over all files in the path
      for file_name in os.listdir(seq_path):
          # Check if the file is a JSON file
          if file_name.endswith(".json"):
              # Extract the sequence ID from the file name
              seq_id = file_name.split(".")[0]
              # Read the JSON file
              with open(os.path.join(seq_path, file_name), "r") as f:
                  content = json.load(f)
              # Append the content to the dictionary
              json_dict[seq_id] = content # json dict 
              ...