Closed cclauss closed 5 years ago
flake8 testing of https://github.com/diux-dev/cluster on Python 3.7.0
$ flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics
./tmux_backend.py:120:68: F821 undefined name 'cmd' assert False, "Timeout %s exceeded for %s" %(max_wait_sec, cmd) ^ ./launch_ray_async_refactored.py:37:17: F821 undefined name 'os' script_name = os.path.basename(BENCHMARK_URL) ^ ./launch_async_adder.py:343:60: F821 undefined name 'num_instances' instances = cluster_aws.CreateAwsInstances(num_instances=num_instances, ^ ./launch_async_adder.py:350:60: F821 undefined name 'INSTANCE_TYPE' instance_type=INSTANCE_TYPE) ^ ./launch_async_adder.py:351:9: F821 undefined name 'AwsJob' job = AwsJob() ^ ./launch_async_adder.py:421:14: F821 undefined name 'generate_tf_env_setup_cmd' task.run(generate_tf_env_setup_cmd(cluster_spec, task_spec)) ^ ./launch_async_adder.py:428:14: F821 undefined name 'generate_tf_env_setup_cmd' task.run(generate_tf_env_setup_cmd(cluster_spec, task_spec)) ^ ./launch_async_adder.py:433:60: F821 undefined name 'logdir' task.run('tensorboard --port=%d --logdir=%s'%(task.port, logdir)) ^ ./util.py:381:70: F821 undefined name 'message' assert not messages, "zone %s is broken? Has messages %s"%(zone, message) ^ ./util.py:749:42: F821 undefined name 'stdout_file' t1 = _StreamOutputToStdout(stdout, stdout_file) ^ ./util.py:751:42: F821 undefined name 'stderr_file' t2 = _StreamOutputToStdout(stderr, stderr_file) ^ ./util.py:964:10: F821 undefined name 'dt' return dt.datetime.fromtimestamp(seconds, pytz.timezone(timezone)) ^ ./util.py:964:45: F821 undefined name 'pytz' return dt.datetime.fromtimestamp(seconds, pytz.timezone(timezone)) ^ ./util.py:1097:12: F821 undefined name 'ec2' volume = ec2.create_volume(Size=size, VolumeType=voltype, ^ ./aws_backend.py:601:20: F821 undefined name 'task' contents = task.file_read(cmd_fn_out) ^ ./tf-tools/benchmark/runner/test_cluster_aws.py:39:3: E999 IndentationError: expected an indented block def main(): ^ ./pytorch_distributed_benchmark/bench_p2p_refactored.py:100:35: F821 undefined name 'FLAGS' assert False, "Unknown role "+FLAGS.role ^ ./pytorch_distributed_benchmark/bench_allreduce.py:173:35: F821 undefined name 'FLAGS' assert False, "Unknown role "+FLAGS.role ^ ./pytorch_distributed_benchmark/bench_p2p.py:113:35: F821 undefined name 'FLAGS' assert False, "Unknown role "+FLAGS.role ^ ./benchmarks/scripts/util/convert_csv_to_json_test.py:31:34: E999 SyntaxError: invalid token datetime.datetime(2017, 06, 26, 2, 59, 29, 325579), ^ ./tf_numpy_benchmark/launch.py:64:35: F821 undefined name 'FLAGS' assert False, "Unknown role "+FLAGS.role ^ ./tf_numpy_benchmark/launch.py:84:16: F821 undefined name 'ami_dict_amazon' ami_dict = ami_dict_amazon ^ ./yuxin_numpy/variable_fetch_bug_report.py:144:15: F821 undefined name 'align_numpy_cpu' params0 = align_numpy_cpu(params0) ^ ./yuxin_numpy/variable_fetch_bug_report.py:146:15: F821 undefined name 'align_numpy_gpu' params0 = align_numpy_gpu(params0) ^ ./yuxin_numpy/variable_fetch_bug_report.py:148:47: F821 undefined name 'create_net' loss, params, grad_cached, grad_assign_op = create_net('net1', params0) ^ ./gpubox/launch_refactored.py:57:26: F821 undefined name 'module_path' local_config_fn = f'{module_path}/jupyter_notebook_config.py' ^ ./gpubox/launch_refactored.py:69:20: F821 undefined name 'module_path' task.upload(f'{module_path}/sample.ipynb', '/efs/notebooks/sample.ipynb', ^ ./gpubox/launch_refactored.py:73:49: F821 undefined name 'job' print(f'Jupyter notebook will be at http://{job.public_ip}:8888') ^ ./gpubox/launch_refactored.py:133:35: F821 undefined name 'FLAGS' assert False, "Unknown role "+FLAGS.role ^ ./gpubox/launch_userdata_test.py:44:35: F821 undefined name 'FLAGS' assert False, "Unknown role "+FLAGS.role ^ ./gpubox/jupyter_notebook_config.py:6369:1: F821 undefined name 'c' c.NotebookApp.ip = '*' ^ ./gpubox/jupyter_notebook_config.py:8321:1: F821 undefined name 'c' c.NotebookApp.password = 'sha1:354b4a71198e:3f1e792c4e9d4f181ac24521cda8dda18a48cbb4' ^ ./gpubox/jupyter_notebook_config.py:8322:1: F821 undefined name 'c' c.NotebookApp.open_browser = False ^ ./gpubox/jupyter_notebook_config.py:8674:1: F821 undefined name 'c' c.NotebookApp.port = 8888 ^ ./gpubox/launch_prewarming_test.py:31:5: F821 undefined name 'worker' worker() ^ ./gpubox/launch_prewarming_test.py:33:35: F821 undefined name 'FLAGS' assert False, "Unknown role "+FLAGS.role ^ ./gpubox/launch.py:140:35: F821 undefined name 'FLAGS' assert False, "Unknown role "+FLAGS.role ^ ./pytorch/jupyter_notebook_config.py:200:1: F821 undefined name 'c' c.NotebookApp.ip = '*' ^ ./pytorch/jupyter_notebook_config.py:261:1: F821 undefined name 'c' c.NotebookApp.password = 'sha1:7987bd0ceb91:50391402d3d581b8c60c9ab65040559580553d85' ^ ./pytorch/jupyter_notebook_config.py:262:1: F821 undefined name 'c' c.NotebookApp.open_browser = False ^ ./pytorch/jupyter_notebook_config.py:273:1: F821 undefined name 'c' c.NotebookApp.port = 8888 ^ ./pytorch/launch_monitoring_refactored.py:33:16: E999 SyntaxError: invalid syntax async=True) ^ ./pytorch/launch_nv_refactored.py:152:23: E999 SyntaxError: invalid syntax task.run(cmd, async=True) ^ ./psbench/launch_tf_adder_refactored.py:75:3: E999 IndentationError: expected an indented block def main(): ^ ./imagenet18/train.py:182:23: E999 SyntaxError: invalid syntax task.run(cmd, async=True) ^ ./imagenet18/tools/launch_tensorboard.py:9:56: E999 SyntaxError: invalid syntax task.run(f'tensorboard --logdir={task.logdir}/..', async=True) ^ ./imagenet64/launch.py:141:14: F821 undefined name 'boto3' client = boto3.client('ec2') ^ ./imagenet64/launch.py:158:18: F821 undefined name 'util' ssh_client = util.SshToHost(self.hostname, ssh_key=self.ssh_key, username=self.username) ^ ./imagenet64/launch.py:165:25: F821 undefined name 'util' self.ssh_client = util.SshToHost(self.hostname, ssh_key=self.ssh_key, username=self.username) ^ ./imagenet64/launch.py:184:11: F821 undefined name 'boto3' ec2 = boto3.resource('ec2') ^ ./imagenet64/launch.py:192:11: F821 undefined name 'boto3' ec2 = boto3.resource('ec2') ^ ./imagenet64/launch.py:223:5: F821 undefined name 'util' util.ExecuteCommandAndWait( ^ ./imagenet64/launch.py:227:12: F821 undefined name 'util' return util.ExecuteCommandAndReturnStdout(self.reuse_ssh_client(), cmd) ^ ./imagenet64/launch.py:238:12: F821 undefined name 'util' return util.ExecuteCommandAndStreamOutput(self.reuse_ssh_client(), ^ ./imagenet64/launch.py:253:12: F821 undefined name 'util' return util.ExecuteCommandInThread( ^ ./imagenet64/launch.py:280:9: F821 undefined name 'boto3' ec2 = boto3.resource('ec2') ^ ./imagenet64/launch.py:282:5: F821 undefined name 'MaybeCreatePlacementGroup' MaybeCreatePlacementGroup(name=placement_group) ^ ./imagenet64/launch.py:316:15: F821 undefined name 'cluster_aws' instances = cluster_aws.CreateAwsInstances(num_instances=num_instances, ^ ./imagenet64/launch.py:316:60: F821 undefined name 'num_instances' instances = cluster_aws.CreateAwsInstances(num_instances=num_instances, ^ ./imagenet64/launch.py:317:55: F821 undefined name 'AMI' image_id=AMI, ^ ./imagenet64/launch.py:318:55: F821 undefined name 'KEY_NAME' key_name=KEY_NAME, ^ ./imagenet64/launch.py:319:54: F821 undefined name 'KEY_PATH' ssh_key=KEY_PATH, ^ ./imagenet64/launch.py:320:61: F821 undefined name 'SECURITY_GROUP' security_group=SECURITY_GROUP, ^ ./imagenet64/launch.py:323:60: F821 undefined name 'INSTANCE_TYPE' instance_type=INSTANCE_TYPE) ^ ./imagenet64/launch.py:324:9: F821 undefined name 'AwsJob' job = AwsJob() ^ ./imagenet64/launch.py:388:14: F821 undefined name 'generate_tf_env_setup_cmd' task.run(generate_tf_env_setup_cmd(cluster_spec, task_spec)) ^ ./imagenet64/launch.py:389:14: F821 undefined name 'PS_CMD' task.run(PS_CMD) ^ ./imagenet64/launch.py:395:14: F821 undefined name 'generate_tf_env_setup_cmd' task.run(generate_tf_env_setup_cmd(cluster_spec, task_spec)) ^ ./imagenet64/launch.py:396:14: F821 undefined name 'WORKER_CMD' task.run(WORKER_CMD) ^ ./imagenet64/launch.py:400:60: F821 undefined name 'logdir' task.run('tensorboard --port=%d --logdir=%s'%(task.port, logdir)) ^ ./imagenet64/launch.py:418:12: F821 undefined name 'launch_job_tmux' ps_job = launch_job_tmux('ps', FLAGS.num_ps) ^ ./imagenet64/launch.py:419:16: F821 undefined name 'launch_job_tmux' worker_job = launch_job_tmux('worker', FLAGS.num_workers) ^ ./imagenet64/launch.py:420:12: F821 undefined name 'launch_job_tmux' tb_job = launch_job_tmux('tb', 1) ^ ./imagenet64/launch.py:438:14: F821 undefined name 'PS_CMD' task.run(PS_CMD) ^ ./imagenet64/launch.py:445:14: F821 undefined name 'WORKER_CMD' task.run(WORKER_CMD) ^ ./pytorch-cifar/launch_refactored.py:49:3: E999 IndentationError: expected an indented block def launch_jupyter(job, sess='jupyter'): ^ ./pytorch-cifar/train_cifar10.py:218:56: E999 SyntaxError: invalid syntax self.next_input = self.next_input.cuda(async=True) ^ ./pytorch-cifar/train_cifar10_bkj.py:78:56: E999 SyntaxError: invalid syntax self.next_input = self.next_input.cuda(async=True) ^ ./pytorch-cifar/train_cifar10_fastai.py:171:56: E999 SyntaxError: invalid syntax self.next_input = self.next_input.cuda(async=True) ^ ./pytorch-cifar/resnet18.py:6:19: F821 undefined name 'nn' class PreActBlock(nn.Module): ^ ./pytorch-cifar/resnet18.py:11:22: F821 undefined name 'nn' self.bn1 = nn.BatchNorm2d(in_channels) ^ ./pytorch-cifar/resnet18.py:12:22: F821 undefined name 'nn' self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False) ^ ./pytorch-cifar/resnet18.py:13:22: F821 undefined name 'nn' self.bn2 = nn.BatchNorm2d(out_channels) ^ ./pytorch-cifar/resnet18.py:14:22: F821 undefined name 'nn' self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False) ^ ./pytorch-cifar/resnet18.py:17:29: F821 undefined name 'nn' self.shortcut = nn.Sequential( ^ ./pytorch-cifar/resnet18.py:18:17: F821 undefined name 'nn' nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False) ^ ./pytorch-cifar/resnet18.py:22:15: F821 undefined name 'F' out = F.relu(self.bn1(x)) ^ ./pytorch-cifar/resnet18.py:25:26: F821 undefined name 'F' out = self.conv2(F.relu(self.bn2(out))) ^ ./pytorch-cifar/resnet18.py:29:16: F821 undefined name 'nn' class ResNet18(nn.Module): ^ ./pytorch-cifar/resnet18.py:35:21: F821 undefined name 'nn' self.prep = nn.Sequential( ^ ./pytorch-cifar/resnet18.py:36:13: F821 undefined name 'nn' nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False), ^ ./pytorch-cifar/resnet18.py:37:13: F821 undefined name 'nn' nn.BatchNorm2d(64), ^ ./pytorch-cifar/resnet18.py:38:13: F821 undefined name 'nn' nn.ReLU() ^ ./pytorch-cifar/resnet18.py:41:23: F821 undefined name 'nn' self.layers = nn.Sequential( ^ ./pytorch-cifar/resnet18.py:48:27: F821 undefined name 'nn' self.classifier = nn.Linear(512, num_classes) ^ ./pytorch-cifar/resnet18.py:58:16: F821 undefined name 'nn' return nn.Sequential(*layers) ^ ./pytorch-cifar/resnet18.py:66:17: F821 undefined name 'F' x_avg = F.adaptive_avg_pool2d(x, (1, 1)) ^ ./pytorch-cifar/resnet18.py:69:17: F821 undefined name 'F' x_max = F.adaptive_max_pool2d(x, (1, 1)) ^ ./pytorch-cifar/resnet18.py:72:13: F821 undefined name 'torch' x = torch.cat([x_avg, x_max], dim=-1) ^ ./iperf/launch.py:109:16: F821 undefined name 'ami_dict_amazon' ami_dict = ami_dict_amazon ^ ./iperf/launch.py:137:9: F821 undefined name 'job' print(job.connect_instructions) ^ 11 E999 SyntaxError: invalid token 90 F821 undefined name 'task' 101
This project exists to keep history of imagenet/ncluster repos, it's not intended to be functional
flake8 testing of https://github.com/diux-dev/cluster on Python 3.7.0
$ flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics