changes to enable multi-ob pred

.gitignore

@@ -4,4 +4,5 @@ tools/Kitti2Coco/labels2coco/*
 tools/Kitti2Coco/trainvalno5k-kitti.txt
 *.pkl
 dino/
-instances_train2014_sel20k.json
\ No newline at end of file
+instances_train2014_sel20k.json
+datasets/**
\ No newline at end of file

docker/build.sh

@@ -11,5 +11,6 @@ docker build . \
     --build-arg NUSC_ROOT=${NUSC_ROOT} \
     --build-arg CADC_ROOT=${CADC_ROOT} \
     --build-arg COCO_ROOT=${COCO_ROOT} \
+    --build-arg VOC_ROOT=${VOC_ROOT} \
     --build-arg LOGDIR=${LOGDIR} \
     -t lost-ece613

docker/config.sh

@@ -14,6 +14,7 @@ LOST_ROOT=/root/lost
 NUSC_ROOT=/root/nusc
 CADC_ROOT=/root/cadc
 COCO_ROOT=/root/coco
+VOC_ROOT=/root/voc
 LOGDIR=/root/logdir
 
 # Workspace structure on host machine
@@ -21,4 +22,5 @@ HOST_LOST_ROOT=/home/sdelcore/wiselab/ece613-lost
 HOST_NUSC_ROOT=/home/sdelcore/datasets/nuscenes
 HOST_CADC_ROOT=/home/sdelcore/datasets/cadc
 HOST_COCO_ROOT=/home/sdelcore/datasets/coco
+HOST_VOC_ROOT=/home/sdelcore/datasets/voc
 HOST_LOGDIR=/home/sdelcore/datasets/logdir
\ No newline at end of file

docker/run.sh

@@ -10,6 +10,7 @@ docker run \
     -v "${HOST_NUSC_ROOT}":"${NUSC_ROOT}" \
     -v "${HOST_CADC_ROOT}":"${CADC_ROOT}" \
     -v "${HOST_COCO_ROOT}":"${COCO_ROOT}" \
+    -v "${HOST_VOC_ROOT}":"${VOC_ROOT}" \
     -v "${HOST_LOGDIR}":"${LOGDIR}" \
     -e DISPLAY=$DISPLAY \
     -e PUID=1000 \

main_lost.py

@@ -115,7 +115,7 @@ if __name__ == "__main__":
     parser.add_argument("--no_hard", action="store_true", help="Only used in the case of the VOC_all setup (see the paper).")
     parser.add_argument("--no_evaluation", action="store_true", help="Compute the evaluation.")
     parser.add_argument("--save_predictions", default=True, type=bool, help="Save predicted bouding boxes.")
-    parser.add_argument("--num_init_seeds", default=50, type=int, help="Number of initial seeds to expand from.")
+    parser.add_argument("--num_init_seeds", default=1, type=int, help="Number of initial seeds to expand from.")
 
     # Visualization
     parser.add_argument(
@@ -152,7 +152,7 @@ if __name__ == "__main__":
 
     if args.image_path is not None:
         args.save_predictions = False
-        args.no_evaluation = False
+        args.no_evaluation = True
         args.dataset = None
 
     # -------------------------------------------------------------------------------------------------------
@@ -207,7 +207,7 @@ if __name__ == "__main__":
 
     total_true_positives = []
     total_false_positives = []
-    total_gt_boxes = 0;
+    total_gt_boxes = 0
 
     pbar = tqdm(dataset.dataloader)
     for im_id, inp in enumerate(pbar):
@@ -334,7 +334,8 @@ if __name__ == "__main__":
             scales,
             init_image_size,
             k_patches=args.k_patches,
-            num_init_seeds=args.num_init_seeds
+            num_init_seeds=args.num_init_seeds,
+            iou_threshold=0.1
             )
 
             # ------------ Visualizations -------------------------------------------
@@ -377,20 +378,18 @@ if __name__ == "__main__":
         nd = len(gt_bbxs)
         total_gt_boxes += nd
 
-        tp = [0] * nd
-        fp = [0] * nd
+        gt_box_mask = [0] * nd
 
-        for idx, gt in enumerate(gt_bbxs):
-            for idy, pred in enumerate(preds):
-                iou = bbox_iou(torch.from_numpy(pred), torch.from_numpy(gt))
-                if iou >= 0.50:
-                    tp[idx] = 1
-                    break
-            else:
-                fp[idx] = 1
-
-        total_true_positives.extend(tp)
-        total_false_positives.extend(fp)
+        for idx, pred in enumerate(preds):
+            ious = bbox_iou(torch.from_numpy(pred), torch.from_numpy(gt_bbxs))
+            for gt_idx, iou in enumerate(ious):
+                if iou > 0.50 and gt_box_mask[gt_idx] == 0:
+                    total_true_positives.append(1)
+                    total_false_positives.append(0)
+                    gt_box_mask[gt_idx] = 1
+                else:
+                    total_true_positives.append(0)
+                    total_false_positives.append(1)
 
     # compute precision recall
     total_false_positives = np.cumsum(total_false_positives)
@@ -399,9 +398,9 @@ if __name__ == "__main__":
     # avoid divide by zero in case the first detection matches a difficult
     # ground truth
     prec = total_true_positives / np.maximum(total_true_positives + total_false_positives, np.finfo(np.float64).eps)
-    ap = voc_ap(rec, prec, use_07_metric=True)
+    ap = voc_ap(rec, prec, use_07_metric=False)
 
-    print("AP: %f" % ap)
+    print("\nAP50: %f" % ap)
 
     # Save predicted bounding boxes
     if args.save_predictions:
@@ -413,9 +412,9 @@ if __name__ == "__main__":
         print("Predictions saved at %s" % filename)
 
     # Evaluate
-    if not args.no_evaluation:
-        print(f"corloc: {100*np.sum(corloc)/cnt:.2f} ({int(np.sum(corloc))}/{cnt})")
-        result_file = os.path.join(folder, 'results.txt')
-        with open(result_file, 'w') as f:
-            f.write('corloc,%.1f,,\n'%(100*np.sum(corloc)/cnt))
-        print('File saved at %s'%result_file)
+    #if not args.no_evaluation:
+    #    print(f"corloc: {100*np.sum(corloc)/cnt:.2f} ({int(np.sum(corloc))}/{cnt})")
+    #    result_file = os.path.join(folder, 'results.txt')
+    #    with open(result_file, 'w') as f:
+    #        f.write('corloc,%.1f,,\n'%(100*np.sum(corloc)/cnt))
+    #    print('File saved at %s'%result_file)

object_discovery.py

@@ -61,6 +61,8 @@ def lost(feats, dims, scales, init_image_size, k_patches=100, num_init_seeds=1,
 
     if num_init_seeds <= 0:
         num_init_seeds = len(sorted_patches)
+    
+    num_init_seeds = min(num_init_seeds, len(sorted_patches))
 
     preds = []
     filtered_seeds= []
@@ -83,6 +85,9 @@ def lost(feats, dims, scales, init_image_size, k_patches=100, num_init_seeds=1,
     for i in range(num_init_seeds):
         add_pred = False
         init_seed = sorted_patches[i]
+
+        if scores[i] > 0:
+            break
         #print('init_seed', init_seed)
         
         potentials = sorted_patches[i:k_patches+i]
@@ -102,6 +107,7 @@ def lost(feats, dims, scales, init_image_size, k_patches=100, num_init_seeds=1,
         )
         pred = np.asarray(pred)
 
+
         #add_pred = aspect_ratio(pred) > 1.0 # TODO Remove assumption
         ious = 0
         
@@ -110,22 +116,8 @@ def lost(feats, dims, scales, init_image_size, k_patches=100, num_init_seeds=1,
         if len(preds) > 0:
             idx_to_remove = -1
             ious, inter, union = _bbox_iou(torch.from_numpy(pred), torch.from_numpy(np.asarray(preds)))
+            add_pred = not any(ious >= iou_threshold)
 
-            for i, p in enumerate(preds) :
-                if box_area(pred) == union[i] or box_area(pred) > box_area(p): #then new prediction is encasing the whole thing so we DO NOT want to add it
-                    #add_pred = False
-                    break 
-                elif box_area(p) == union[i] or box_area(pred) < box_area(p): #then this already stored pred is larger so we want to remove it and add pred
-                    add_pred= True
-                    #idx_to_remove = i
-                    #print('elim bbox: ', p, 'add: ', pred)
-                    break 
-            if idx_to_remove >= 0:
-                #print(preds, idx_to_remove)
-                preds.pop(idx_to_remove)
-                ious = ious[ious!=ious[idx_to_remove]] 
-
-            add_pred = add_pred and not any(ious >= iou_threshold)
         else:
             add_pred = True
             

scripts/run-coco.sh

+
+OUTPUT_PATH=/root/lost/datasets/COCO/outputs/coco
+
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=16
+NUM_INIT_SEEDS=100
+
+cd /root/lost/
+rm -rf $OUTPUT_PATH
+mkdir -p $OUTPUT_PATH
+echo $OUTPUT_PATH
+
+python main_lost.py \
+    --dataset COCO20k \
+    --set train \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --visualize pred \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+exit
\ No newline at end of file

scripts/run-dataset.sh

 
-OUTPUT_PATH=/root/lost/outputs/coco
+OUTPUT_PATH=/root/voc/outputs/voc
 
 DINO_ARCH=vit_base
 LOST_FEATURES=k
-K_PATCHES=25
+K_PATCHES=100
 PATCH_SIZE=16
+NUM_INIT_SEEDS=100
 
 cd /root/lost/
 rm -rf $OUTPUT_PATH
@@ -12,14 +13,14 @@ mkdir -p $OUTPUT_PATH
 echo $OUTPUT_PATH
 
 python main_lost.py \
-    --dataset COCO20k \
-    --set train \
+    --dataset VOC07 \
+    --set trainval \
     --output_dir $OUTPUT_PATH \
     --arch $DINO_ARCH \
     --which_feature $LOST_FEATURES \
     --k_patches $K_PATCHES \
     --patch_size $PATCH_SIZE \
     --visualize pred \
-    --num_init_seeds 1
+    --num_init_seeds $NUM_INIT_SEEDS
 
 exit
\ No newline at end of file

scripts/run-single-image.sh

 declare -a images=(
-    "COCO_train2014_000000000077" # SOME IMAGE NAME
+    "000021" "000041" "000005" # SOME IMAGE NAME
 )
 
-DATASET_PATH=$COCO_ROOT/images/train2014
+DATASET_PATH=/root/lost/datasets/VOC2007/VOCdevkit/VOC2007/JPEGImages
 LOST_PATH=~/lost
 DINO_PATH=$LOST_PATH/dino
 OUTPUT_PATH=$LOST_PATH/outputs/samples

scripts/run-tests-coco.sh

+
+OUTPUT_PATH=/root/lost/datasets/COCO/outputs/coco
+
+
+
+cd /root/lost/
+rm -rf $OUTPUT_PATH
+mkdir -p $OUTPUT_PATH
+echo $OUTPUT_PATH
+
+echo "COCO vit_base k=100 p=16, num_init_seed=1"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=16
+NUM_INIT_SEEDS=1
+
+python main_lost.py \
+    --dataset COCO20k \
+    --set train \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "COCO vit_base k=100 p=16, num_init_seed=100"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=16
+NUM_INIT_SEEDS=100
+
+python main_lost.py \
+    --dataset COCO20k \
+    --set train \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "COCO vit_small k=100 p=16, num_init_seed=100"
+DINO_ARCH=vit_small
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=16
+NUM_INIT_SEEDS=100
+
+python main_lost.py \
+    --dataset COCO20k \
+    --set train \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "COCO vit_base k=50 p=16, num_init_seed=100"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=50
+PATCH_SIZE=16
+NUM_INIT_SEEDS=100
+
+python main_lost.py \
+    --dataset COCO20k \
+    --set train \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "COCO vit_base k=150 p=16, num_init_seed=100"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=150
+PATCH_SIZE=16
+NUM_INIT_SEEDS=100
+
+python main_lost.py \
+    --dataset COCO20k \
+    --set train \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "COCO vit_base k=100 p=8, num_init_seed=100"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=8
+NUM_INIT_SEEDS=100
+
+python main_lost.py \
+    --dataset COCO20k \
+    --set train \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "COCO vit_base k=100 p=16, num_init_seed=25"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=16
+NUM_INIT_SEEDS=25
+
+python main_lost.py \
+    --dataset COCO20k \
+    --set train \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
\ No newline at end of file

scripts/run-tests-voc.sh

+
+OUTPUT_PATH=/root/voc/outputs/voc
+
+
+
+cd /root/lost/
+rm -rf $OUTPUT_PATH
+mkdir -p $OUTPUT_PATH
+echo $OUTPUT_PATH
+
+echo "VOC vit_base k=100 p=16, num_init_seed=1"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=16
+NUM_INIT_SEEDS=1
+
+python main_lost.py \
+    --dataset VOC07 \
+    --set trainval \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "VOC vit_base k=100 p=16, num_init_seed=100"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=16
+NUM_INIT_SEEDS=100
+
+python main_lost.py \
+    --dataset VOC07 \
+    --set trainval \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "VOC vit_small k=100 p=16, num_init_seed=100"
+DINO_ARCH=vit_small
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=16
+NUM_INIT_SEEDS=100
+
+python main_lost.py \
+    --dataset VOC07 \
+    --set trainval \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "VOC vit_base k=50 p=16, num_init_seed=100"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=50
+PATCH_SIZE=16
+NUM_INIT_SEEDS=100
+
+python main_lost.py \
+    --dataset VOC07 \
+    --set trainval \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "VOC vit_base k=150 p=16, num_init_seed=100"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=150
+PATCH_SIZE=16
+NUM_INIT_SEEDS=100
+
+python main_lost.py \
+    --dataset VOC07 \
+    --set trainval \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "VOC vit_base k=100 p=8, num_init_seed=100"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=8
+NUM_INIT_SEEDS=100
+
+python main_lost.py \
+    --dataset VOC07 \
+    --set trainval \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
+
+echo "VOC vit_base k=100 p=16, num_init_seed=25"
+DINO_ARCH=vit_base
+LOST_FEATURES=k
+K_PATCHES=100
+PATCH_SIZE=16
+NUM_INIT_SEEDS=25
+
+python main_lost.py \
+    --dataset VOC07 \
+    --set trainval \
+    --output_dir $OUTPUT_PATH \
+    --arch $DINO_ARCH \
+    --which_feature $LOST_FEATURES \
+    --k_patches $K_PATCHES \
+    --patch_size $PATCH_SIZE \
+    --num_init_seeds $NUM_INIT_SEEDS
\ No newline at end of file