add WebVision dataset

Author: AgentDS

README.md

@@ -138,6 +138,15 @@ Raw dataset should be downloaded in to local folder before data-build process. T
 │   │           ├── t10k-images-idx3-ubyte
 │   │           ...
 │   │           └── train-labels-idx1-ubyte.gz
+│   ├── webvision1.0/
+│   │   ├── info
+│   │   ├── google
+│   │   ├── README.txt
+│   │   └── val_images_256
+│   ├── imagenet_data/
+│   │   └── ILSVRC2012
+│   │       ├── train
+│   │       └── val
 │   ├── clothing1M/
 │   │   ├── category_names_chn.txt
 │   │   ├── category_names_eng.txt
@@ -182,6 +191,8 @@ Raw dataset should be downloaded in to local folder before data-build process. T
 
 - To download __[Clothing1M](https://github.com/Cysu/noisy_label)__, please contact *tong.xiao.work[at]gmail[dot]com* to get the download link. Untar the images and unzip the annotations under `rawdata/clothing1M`.
 
+- To download __[WebVision1.0](https://data.vision.ee.ethz.ch/cvl/webvision/download.html)__, only use "Resized Images (small version)". Unzip the data under `rawdata/webvision1.0`.
+
 
 
 ### Build dataset with noisy label
@@ -216,10 +227,13 @@ $ python build_dataset_fed.py --dataset cifar10 \
   - `--min_noise_ratio 0.3 --max_noise_ratio 0.5 --noise_mode sym` for localized symmetric noise $\varepsilon_k \sim \mathcal{U}(0.3, 0.5)$
   - `--min_noise_ratio 0.3 --max_noise_ratio 0.5 --noise_mode asym` for localized asymmetric noise $\varepsilon_k \sim \mathcal{U}(0.3, 0.5)$
 
-- Real noise (only works for Clothing1M): `--dataset clothing1m --globalize --noise_mode real --num_sampels 64000`
-
-  - `--num_samples` is for specifying number of training samples used for Clothing1M, the default is 64000
+- Real noise:
 
+  - `--dataset clothing1m` for Clothing1M, need to specify ` --num_sampels` as number of training samples used for Clothing1M, the default is 64000
+  - `--dataset webvision` for WebVision1.0, need to specify `--raw_imagenet_dir` for ImageNet validation set raw data dir
+  
+  > Once `--dataset` is set with `clothing1m` or `webvision`, the arguement will automatically set `--globalize --noise_mode real` for real world noise.
+  
 
 
 
@@ -244,6 +258,12 @@ $ python build_dataset_fed.py --dataset cifar10 \
   - Non-IID quantity skew: `--partition noniid-quantity --num_clients 10 --dir_alpha 0.1`
   - Non-IID Dirichlet-based label skew: `--partition noniid-labeldir --dir_alpha 0.1 --num_clients 10`
   - Non-IID quantity-based label skew: `--partition noniid-#label --major_classes_num 5 --num_clients 10`
+- WebVision: `--dataset webvision`
+  - IID: `--partition iid --num_clients 10`
+  - Non-IID quantity skew: `--partition noniid-quantity --num_clients 10 --dir_alpha 0.1`
+  - Non-IID Dirichlet-based label skew: `--partition noniid-labeldir --dir_alpha 0.6 --num_clients 10`
+  - Non-IID quantity-based label skew: `--partition noniid-#label --major_classes_num 20 --num_clients 10`
+
 
 
 
@@ -544,6 +564,7 @@ For more scripts, please check [scripts](./scripts/) folder.
 ### Datasets
 
 - [ ] Include mini-ImageNet (synthetic noise)
+- [x] Include WebVision (real-world noise)
 - [ ] Include Food-101N (real-world noise)
 - [ ] Include ANIMAL-10N (real-world noise)
 

build_dataset_fed.py

@@ -44,6 +44,7 @@
     - Non-IID-xxx
     - Non-IID-xxx
 """
+
 import argparse
 
 # from progress.bar import Bar as Bar
@@ -58,7 +59,10 @@
     FedNLLMNIST,
     FedNLLSVHN,
     FedNLLClothing1M,
+    FedNLLWebVision,
+    FedNLLSynthetic,
 )
+from fednoisy.data.NLLData import functional as nllF
 
 
 def read_args():
@@ -74,9 +78,25 @@ def read_args():
         "--partition",
         default="iid",
         type=str,
-        choices=["iid", "noniid-#label", "noniid-labeldir", "noniid-quantity"],
+        choices=[
+            "iid",
+            "noniid",
+            "noniid-#label",
+            "noniid-labeldir",
+            "noniid-quantity",
+        ],
         help="Data partition scheme for federated setting.",
     )
+    parser.add_argument(
+        "--personalize",
+        action="store_true",
+        help="Whether use personalized local test set for each client. If True, then each client's class ratio of local test set is same as the training set",
+    )
+    parser.add_argument(
+        "--balance",
+        action="store_true",
+        help="whether use balance partition for Synthetic dataset.",
+    )
     parser.add_argument(
         "--num_clients",
         default=10,
@@ -140,24 +160,56 @@ def read_args():
         "--num_samples",
         default=32 * 2 * 1000,
         type=int,
-        help="Number of samples used for Clothing1M training. Defaults as 64000.",
+        help="Number of samples used for Clothing1M/Synthetic data training. Defaults as 64000.",
+    )
+
+    parser.add_argument(
+        "--num_test_samples",
+        default=1000,
+        type=int,
+        help="Number of test samples for synthetic dataset.",
+    )
+    parser.add_argument(
+        "--feature_dim",
+        type=int,
+        default=100,
+        help="Feature dimension for synthetic dataset.",
+    )
+    parser.add_argument(
+        "--use_bias",
+        action="store_true",
+        help="Whether to use bias in synthetic data generation. If True, Y = Xw + b + ε; otherwise Y = Xw + ε.",
     )
 
     # ----Dataset path options----
     parser.add_argument(
         "--dataset",
         default="cifar10",
         type=str,
-        choices=["mnist", "cifar10", "cifar100", "svhn", "clothing1m", "webvision"],
+        choices=[
+            "mnist",
+            "cifar10",
+            "cifar100",
+            "svhn",
+            "clothing1m",
+            "webvision",
+            "synthetic",
+        ],
         help="Dataset for experiment. Current support: ['mnist', 'cifar10', "
-        "'cifar100', 'svhn', 'clothing1m', 'webvision']",
+        "'cifar100', 'svhn', 'clothing1m', 'webvision', 'synthetic]",
     )
     parser.add_argument(
         "--raw_data_dir",
         default="../data",
         type=str,
         help="Directory for raw dataset download",
     )
+    parser.add_argument(
+        "--raw_imagenet_dir",
+        default="../rawdata/imagenet",
+        type=str,
+        help="Directory for raw dataset download",
+    )
     parser.add_argument(
         "--data_dir",
         default="../noisy_label_data",
@@ -242,9 +294,11 @@ def read_args():
             max_noise_ratio=args.max_noise_ratio,
             root_dir=args.raw_data_dir,
             out_dir=args.data_dir,
+            personalize=args.personalize,
         )
         nll_cifar10.create_nll_scene(seed=args.seed)
         nll_cifar10.save_nll_scene()
+
     elif args.dataset == "cifar100":
         nll_cifar100 = FedNLLCIFAR100(
             globalize=args.globalize,
@@ -258,9 +312,11 @@ def read_args():
             max_noise_ratio=args.max_noise_ratio,
             root_dir=args.raw_data_dir,
             out_dir=args.data_dir,
+            personalize=args.personalize,
         )
         nll_cifar100.create_nll_scene(seed=args.seed)
         nll_cifar100.save_nll_scene()
+
     elif args.dataset == "mnist":
         nll_mnist = FedNLLMNIST(
             globalize=args.globalize,
@@ -274,6 +330,7 @@ def read_args():
             max_noise_ratio=args.max_noise_ratio,
             root_dir=args.raw_data_dir,
             out_dir=args.data_dir,
+            personalize=args.personalize,
         )
         nll_mnist.create_nll_scene(seed=args.seed)
         nll_mnist.save_nll_scene()
@@ -291,11 +348,15 @@ def read_args():
             max_noise_ratio=args.max_noise_ratio,
             root_dir=args.raw_data_dir,
             out_dir=args.data_dir,
+            personalize=args.personalize,
         )
         nll_svhn.create_nll_scene(seed=args.seed)
         nll_svhn.save_nll_scene()
 
     elif args.dataset == "clothing1m":
+        args.noise_mode = "real"
+        args.globalize = True
+        args.noise_ratio = 0.39
         nll_clothing1m = FedNLLClothing1M(
             root_dir=args.raw_data_dir,
             out_dir=args.data_dir,
@@ -308,5 +369,42 @@ def read_args():
         nll_clothing1m.create_nll_scene(seed=args.seed)
         nll_clothing1m.save_nll_scene()
 
+    elif args.dataset == "webvision":
+        args.noise_mode = "real"
+        args.globalize = True
+        args.noise_ratio = 0.20
+        nll_webvision = FedNLLWebVision(
+            root_dir=args.raw_data_dir,
+            imagenet_root_dir=args.raw_imagenet_dir,
+            out_dir=args.data_dir,
+            partition=args.partition,
+            num_clients=args.num_clients,
+            dir_alpha=args.dir_alpha,
+            major_classes_num=args.major_classes_num,
+        )
+        nll_webvision.create_nll_scene(seed=args.seed)
+        nll_webvision.save_nll_scene()
+
+    elif args.dataset == "synthetic":
+        nll_synthetic = FedNLLSynthetic(
+            out_dir=args.data_dir,
+            num_clients=args.num_clients,
+            init_mu=0,
+            init_sigma=1,
+            partition=args.partition,
+            balance=args.balance,
+            train_sample_num=args.num_samples,
+            test_sample_num=args.num_test_samples,
+            feature_dim=args.feature_dim,
+            use_bias=args.use_bias,
+            dir_alpha=args.dir_alpha,
+        )
+        args.init_mu = 0
+        args.init_sigma = 1
+        nll_synthetic.create_nll_scene(seed=args.seed)
+        nll_synthetic.save_nll_scene()
+        nll_name = nllF.FedNLL_name(**vars(args))
+        print(f"{nll_name}")
+
     else:
         raise ValueError(f"dataset='{args.dataset}' is not supported!")

fednoisy/algorithms/fedavg/client.py

@@ -107,6 +107,7 @@ def train(self, model_parameters, train_loader):
         data_size = len(train_loader.dataset)
 
         for epoch in range(self.epochs):
+            self._model.train()  # TODO
             self._LOGGER.info(
                 f"Round {self.round} client-{self.cur_cid} local train epoch [{epoch}/{self.epochs}]"
             )
@@ -137,6 +138,8 @@ def evaluate(self):
             multimodel=multimodel,
         )
 
+        # TODO: add ImageNet evaluation code
+
         return loss_, acc_
 
 

fednoisy/algorithms/fedavg/main.py

@@ -55,6 +55,10 @@
     args.noise_mode = "real"
     args.globalize = True
     args.noise_ratio = 0.39
+elif args.dataset == "webvision":
+    args.noise_mode = "real"
+    args.globalize = True
+    args.noise_ratio = 0.20
 
 nll_name = nllF.FedNLL_name(**vars(args))
 exp_name = make_exp_name("fedavg", args)

fednoisy/algorithms/fedavg/misc.py

@@ -29,7 +29,7 @@ def read_fednll_args():
     parser.add_argument("--batch_size", type=int, default=128)
     parser.add_argument("--epochs", type=int, default=2)
     parser.add_argument("--lr", type=float, default=0.01)
-    parser.add_argument("--weight_decay", type=float, default=1e-3)
+    parser.add_argument("--weight_decay", type=float, default=5e-4)
     parser.add_argument("--momentum", type=float, default=0.9)
 
     # ==== FedNLL data args ====

fednoisy/algorithms/singleset/__init__.py

@@ -0,0 +1,93 @@
+import torch
+import argparse
+import sys
+import os
+from copy import deepcopy
+from typing import Dict, Tuple, List, Optional
+
+from torch import nn
+from torch.utils.data import DataLoader
+import torchvision
+import torchvision.transforms as transforms
+
+from fedlab.contrib.dataset.basic_dataset import FedDataset
+from fedlab.utils.logger import Logger
+
+sys.path.append(os.getcwd())
+from fednoisy.data.NLLData import functional as nllF
+from fednoisy.data import (
+    CLASS_NUM,
+    TRAIN_SAMPLE_NUM,
+    TEST_SAMPLE_NUM,
+    CIFAR10_TRANSITION_MATRIX,
+    NORM_VALUES,
+)
+
+# from fednoisy.utils import misc as misc
+from fednoisy.data.NLLData import functional as nllF
+from fednoisy.data.dataset import FedNLLClientDataset
+from fednoisy.utils.misc import setup_seed, make_dirs, make_exp_name, AverageMeter
+from fednoisy.models.build_model import build_model
+from fednoisy.algorithms.singleset.misc import read_singlenll_args
+
+
+args = read_singlenll_args()
+if torch.cuda.is_available():
+    args.cuda = True
+else:
+    args.cuda = False
+
+setup_seed(args.seed)
+nll_name = nllF.FedNLL_name(**vars(args))
+
+# ==== Data loader
+test_transform = transforms.Compose(
+    [
+        transforms.ToTensor(),
+        transforms.Normalize(*NORM_VALUES[args.dataset]),
+    ]
+)
+test_dataset = torchvision.datasets.CIFAR10(
+    train=False, root=args.raw_data_dir, transform=test_transform
+)
+test_loader = DataLoader(
+    dataset=test_dataset, batch_size=args.batch_size, shuffle=False
+)
+dataset = FedNLLClientDataset(args)
+train_loader = dataset.get_dataloader(args.client_id, args.batch_size)
+
+# ==== Get model
+model = build_model(args.model, CLASS_NUM[args.dataset])
+model = model.to(args.device)
+
+# === Optimizer
+# optimizer = torch.optim.SGD(
+#     model.parameters(), args.lr, weight_decay=args.weight_decay, momentum=args.momentum
+# )
+optimizer = torch.optim.SGD(model.parameters(), args.lr, weight_decay=args.weight_decay)
+criterion = torch.nn.CrossEntropyLoss()
+
+# ==== Setup log
+logger = Logger(log_name="SingleSetTrainer")
+
+# ==== Training
+for epoch in range(args.epochs):
+    logger.info(f"Epoch [{epoch}/{args.epochs}] Client-{args.client_id} local training")
+    model.train()
+    for imgs, labels, noisy_labels in train_loader:
+        if args.cuda:
+            imgs = imgs.cuda(args.device)
+            noisy_labels = noisy_labels.cuda(args.device)
+
+        output = model(imgs)
+        loss = criterion(output, noisy_labels)
+
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+        # logger.info(f"loss: {loss.item()}")
+
+    loss_, acc_ = evaluate(model, nn.CrossEntropyLoss(), test_loader)
+    logger.info(
+        f"Epoch [{epoch}/{args.epochs}] Client-{args.client_id} test accuracy: {acc_*100}%"
+    )