search

diux-dev / cluster

train on AWS
75 stars 15 forks source link

Syntax errors and undefined names #72

Closed cclauss closed 5 years ago

cclauss commented 6 years ago

flake8 testing of https://github.com/diux-dev/cluster on Python 3.7.0

$ flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics

./tmux_backend.py:120:68: F821 undefined name 'cmd'
        assert False, "Timeout %s exceeded for %s" %(max_wait_sec, cmd)
                                                                   ^
./launch_ray_async_refactored.py:37:17: F821 undefined name 'os'
  script_name = os.path.basename(BENCHMARK_URL)
                ^
./launch_async_adder.py:343:60: F821 undefined name 'num_instances'
  instances = cluster_aws.CreateAwsInstances(num_instances=num_instances,
                                                           ^
./launch_async_adder.py:350:60: F821 undefined name 'INSTANCE_TYPE'
                                             instance_type=INSTANCE_TYPE)
                                                           ^
./launch_async_adder.py:351:9: F821 undefined name 'AwsJob'
  job = AwsJob()
        ^
./launch_async_adder.py:421:14: F821 undefined name 'generate_tf_env_setup_cmd'
    task.run(generate_tf_env_setup_cmd(cluster_spec, task_spec))
             ^
./launch_async_adder.py:428:14: F821 undefined name 'generate_tf_env_setup_cmd'
    task.run(generate_tf_env_setup_cmd(cluster_spec, task_spec))
             ^
./launch_async_adder.py:433:60: F821 undefined name 'logdir'
  task.run('tensorboard --port=%d --logdir=%s'%(task.port, logdir))
                                                           ^
./util.py:381:70: F821 undefined name 'message'
    assert not messages, "zone %s is broken? Has messages %s"%(zone, message)
                                                                     ^
./util.py:749:42: F821 undefined name 'stdout_file'
      t1 = _StreamOutputToStdout(stdout, stdout_file)
                                         ^
./util.py:751:42: F821 undefined name 'stderr_file'
      t2 = _StreamOutputToStdout(stderr, stderr_file)
                                         ^
./util.py:964:10: F821 undefined name 'dt'
  return dt.datetime.fromtimestamp(seconds, pytz.timezone(timezone))
         ^
./util.py:964:45: F821 undefined name 'pytz'
  return dt.datetime.fromtimestamp(seconds, pytz.timezone(timezone))
                                            ^
./util.py:1097:12: F821 undefined name 'ec2'
  volume = ec2.create_volume(Size=size, VolumeType=voltype,
           ^
./aws_backend.py:601:20: F821 undefined name 'task'
        contents = task.file_read(cmd_fn_out)
                   ^
./tf-tools/benchmark/runner/test_cluster_aws.py:39:3: E999 IndentationError: expected an indented block
def main():
  ^
./pytorch_distributed_benchmark/bench_p2p_refactored.py:100:35: F821 undefined name 'FLAGS'
    assert False, "Unknown role "+FLAGS.role
                                  ^
./pytorch_distributed_benchmark/bench_allreduce.py:173:35: F821 undefined name 'FLAGS'
    assert False, "Unknown role "+FLAGS.role
                                  ^
./pytorch_distributed_benchmark/bench_p2p.py:113:35: F821 undefined name 'FLAGS'
    assert False, "Unknown role "+FLAGS.role
                                  ^
./benchmarks/scripts/util/convert_csv_to_json_test.py:31:34: E999 SyntaxError: invalid token
        datetime.datetime(2017, 06, 26, 2, 59, 29, 325579),
                                 ^
./tf_numpy_benchmark/launch.py:64:35: F821 undefined name 'FLAGS'
    assert False, "Unknown role "+FLAGS.role
                                  ^
./tf_numpy_benchmark/launch.py:84:16: F821 undefined name 'ami_dict_amazon'
    ami_dict = ami_dict_amazon
               ^
./yuxin_numpy/variable_fetch_bug_report.py:144:15: F821 undefined name 'align_numpy_cpu'
    params0 = align_numpy_cpu(params0)
              ^
./yuxin_numpy/variable_fetch_bug_report.py:146:15: F821 undefined name 'align_numpy_gpu'
    params0 = align_numpy_gpu(params0)
              ^
./yuxin_numpy/variable_fetch_bug_report.py:148:47: F821 undefined name 'create_net'
  loss, params, grad_cached, grad_assign_op = create_net('net1', params0)
                                              ^
./gpubox/launch_refactored.py:57:26: F821 undefined name 'module_path'
    local_config_fn = f'{module_path}/jupyter_notebook_config.py'
                         ^
./gpubox/launch_refactored.py:69:20: F821 undefined name 'module_path'
    task.upload(f'{module_path}/sample.ipynb', '/efs/notebooks/sample.ipynb',
                   ^
./gpubox/launch_refactored.py:73:49: F821 undefined name 'job'
    print(f'Jupyter notebook will be at http://{job.public_ip}:8888')
                                                ^
./gpubox/launch_refactored.py:133:35: F821 undefined name 'FLAGS'
    assert False, "Unknown role "+FLAGS.role
                                  ^
./gpubox/launch_userdata_test.py:44:35: F821 undefined name 'FLAGS'
    assert False, "Unknown role "+FLAGS.role
                                  ^
./gpubox/jupyter_notebook_config.py:6369:1: F821 undefined name 'c'
c.NotebookApp.ip = '*'
^
./gpubox/jupyter_notebook_config.py:8321:1: F821 undefined name 'c'
c.NotebookApp.password = 'sha1:354b4a71198e:3f1e792c4e9d4f181ac24521cda8dda18a48cbb4'
^
./gpubox/jupyter_notebook_config.py:8322:1: F821 undefined name 'c'
c.NotebookApp.open_browser = False
^
./gpubox/jupyter_notebook_config.py:8674:1: F821 undefined name 'c'
c.NotebookApp.port = 8888
^
./gpubox/launch_prewarming_test.py:31:5: F821 undefined name 'worker'
    worker()
    ^
./gpubox/launch_prewarming_test.py:33:35: F821 undefined name 'FLAGS'
    assert False, "Unknown role "+FLAGS.role
                                  ^
./gpubox/launch.py:140:35: F821 undefined name 'FLAGS'
    assert False, "Unknown role "+FLAGS.role
                                  ^
./pytorch/jupyter_notebook_config.py:200:1: F821 undefined name 'c'
c.NotebookApp.ip = '*'
^
./pytorch/jupyter_notebook_config.py:261:1: F821 undefined name 'c'
c.NotebookApp.password = 'sha1:7987bd0ceb91:50391402d3d581b8c60c9ab65040559580553d85'
^
./pytorch/jupyter_notebook_config.py:262:1: F821 undefined name 'c'
c.NotebookApp.open_browser = False
^
./pytorch/jupyter_notebook_config.py:273:1: F821 undefined name 'c'
c.NotebookApp.port = 8888
^
./pytorch/launch_monitoring_refactored.py:33:16: E999 SyntaxError: invalid syntax
           async=True)
               ^
./pytorch/launch_nv_refactored.py:152:23: E999 SyntaxError: invalid syntax
    task.run(cmd, async=True)
                      ^
./psbench/launch_tf_adder_refactored.py:75:3: E999 IndentationError: expected an indented block
def main():
  ^
./imagenet18/train.py:182:23: E999 SyntaxError: invalid syntax
    task.run(cmd, async=True)
                      ^
./imagenet18/tools/launch_tensorboard.py:9:56: E999 SyntaxError: invalid syntax
task.run(f'tensorboard --logdir={task.logdir}/..', async=True)
                                                       ^
./imagenet64/launch.py:141:14: F821 undefined name 'boto3'
    client = boto3.client('ec2')
             ^
./imagenet64/launch.py:158:18: F821 undefined name 'util'
    ssh_client = util.SshToHost(self.hostname, ssh_key=self.ssh_key, username=self.username)
                 ^
./imagenet64/launch.py:165:25: F821 undefined name 'util'
      self.ssh_client = util.SshToHost(self.hostname, ssh_key=self.ssh_key, username=self.username)
                        ^
./imagenet64/launch.py:184:11: F821 undefined name 'boto3'
    ec2 = boto3.resource('ec2')
          ^
./imagenet64/launch.py:192:11: F821 undefined name 'boto3'
    ec2 = boto3.resource('ec2')
          ^
./imagenet64/launch.py:223:5: F821 undefined name 'util'
    util.ExecuteCommandAndWait(
    ^
./imagenet64/launch.py:227:12: F821 undefined name 'util'
    return util.ExecuteCommandAndReturnStdout(self.reuse_ssh_client(), cmd)
           ^
./imagenet64/launch.py:238:12: F821 undefined name 'util'
    return util.ExecuteCommandAndStreamOutput(self.reuse_ssh_client(),
           ^
./imagenet64/launch.py:253:12: F821 undefined name 'util'
    return util.ExecuteCommandInThread(
           ^
./imagenet64/launch.py:280:9: F821 undefined name 'boto3'
  ec2 = boto3.resource('ec2')
        ^
./imagenet64/launch.py:282:5: F821 undefined name 'MaybeCreatePlacementGroup'
    MaybeCreatePlacementGroup(name=placement_group)
    ^
./imagenet64/launch.py:316:15: F821 undefined name 'cluster_aws'
  instances = cluster_aws.CreateAwsInstances(num_instances=num_instances,
              ^
./imagenet64/launch.py:316:60: F821 undefined name 'num_instances'
  instances = cluster_aws.CreateAwsInstances(num_instances=num_instances,
                                                           ^
./imagenet64/launch.py:317:55: F821 undefined name 'AMI'
                                             image_id=AMI,
                                                      ^
./imagenet64/launch.py:318:55: F821 undefined name 'KEY_NAME'
                                             key_name=KEY_NAME,
                                                      ^
./imagenet64/launch.py:319:54: F821 undefined name 'KEY_PATH'
                                             ssh_key=KEY_PATH,
                                                     ^
./imagenet64/launch.py:320:61: F821 undefined name 'SECURITY_GROUP'
                                             security_group=SECURITY_GROUP,
                                                            ^
./imagenet64/launch.py:323:60: F821 undefined name 'INSTANCE_TYPE'
                                             instance_type=INSTANCE_TYPE)
                                                           ^
./imagenet64/launch.py:324:9: F821 undefined name 'AwsJob'
  job = AwsJob()
        ^
./imagenet64/launch.py:388:14: F821 undefined name 'generate_tf_env_setup_cmd'
    task.run(generate_tf_env_setup_cmd(cluster_spec, task_spec))
             ^
./imagenet64/launch.py:389:14: F821 undefined name 'PS_CMD'
    task.run(PS_CMD)
             ^
./imagenet64/launch.py:395:14: F821 undefined name 'generate_tf_env_setup_cmd'
    task.run(generate_tf_env_setup_cmd(cluster_spec, task_spec))
             ^
./imagenet64/launch.py:396:14: F821 undefined name 'WORKER_CMD'
    task.run(WORKER_CMD)
             ^
./imagenet64/launch.py:400:60: F821 undefined name 'logdir'
  task.run('tensorboard --port=%d --logdir=%s'%(task.port, logdir))
                                                           ^
./imagenet64/launch.py:418:12: F821 undefined name 'launch_job_tmux'
  ps_job = launch_job_tmux('ps', FLAGS.num_ps)
           ^
./imagenet64/launch.py:419:16: F821 undefined name 'launch_job_tmux'
  worker_job = launch_job_tmux('worker', FLAGS.num_workers)
               ^
./imagenet64/launch.py:420:12: F821 undefined name 'launch_job_tmux'
  tb_job = launch_job_tmux('tb', 1)
           ^
./imagenet64/launch.py:438:14: F821 undefined name 'PS_CMD'
    task.run(PS_CMD)
             ^
./imagenet64/launch.py:445:14: F821 undefined name 'WORKER_CMD'
    task.run(WORKER_CMD)
             ^
./pytorch-cifar/launch_refactored.py:49:3: E999 IndentationError: expected an indented block
def launch_jupyter(job, sess='jupyter'):
  ^
./pytorch-cifar/train_cifar10.py:218:56: E999 SyntaxError: invalid syntax
            self.next_input = self.next_input.cuda(async=True)
                                                       ^
./pytorch-cifar/train_cifar10_bkj.py:78:56: E999 SyntaxError: invalid syntax
            self.next_input = self.next_input.cuda(async=True)
                                                       ^
./pytorch-cifar/train_cifar10_fastai.py:171:56: E999 SyntaxError: invalid syntax
            self.next_input = self.next_input.cuda(async=True)
                                                       ^
./pytorch-cifar/resnet18.py:6:19: F821 undefined name 'nn'
class PreActBlock(nn.Module):
                  ^
./pytorch-cifar/resnet18.py:11:22: F821 undefined name 'nn'
        self.bn1   = nn.BatchNorm2d(in_channels)
                     ^
./pytorch-cifar/resnet18.py:12:22: F821 undefined name 'nn'
        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)
                     ^
./pytorch-cifar/resnet18.py:13:22: F821 undefined name 'nn'
        self.bn2   = nn.BatchNorm2d(out_channels)
                     ^
./pytorch-cifar/resnet18.py:14:22: F821 undefined name 'nn'
        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)
                     ^
./pytorch-cifar/resnet18.py:17:29: F821 undefined name 'nn'
            self.shortcut = nn.Sequential(
                            ^
./pytorch-cifar/resnet18.py:18:17: F821 undefined name 'nn'
                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False)
                ^
./pytorch-cifar/resnet18.py:22:15: F821 undefined name 'F'
        out = F.relu(self.bn1(x))
              ^
./pytorch-cifar/resnet18.py:25:26: F821 undefined name 'F'
        out = self.conv2(F.relu(self.bn2(out)))
                         ^
./pytorch-cifar/resnet18.py:29:16: F821 undefined name 'nn'
class ResNet18(nn.Module):
               ^
./pytorch-cifar/resnet18.py:35:21: F821 undefined name 'nn'
        self.prep = nn.Sequential(
                    ^
./pytorch-cifar/resnet18.py:36:13: F821 undefined name 'nn'
            nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False),
            ^
./pytorch-cifar/resnet18.py:37:13: F821 undefined name 'nn'
            nn.BatchNorm2d(64),
            ^
./pytorch-cifar/resnet18.py:38:13: F821 undefined name 'nn'
            nn.ReLU()
            ^
./pytorch-cifar/resnet18.py:41:23: F821 undefined name 'nn'
        self.layers = nn.Sequential(
                      ^
./pytorch-cifar/resnet18.py:48:27: F821 undefined name 'nn'
        self.classifier = nn.Linear(512, num_classes)
                          ^
./pytorch-cifar/resnet18.py:58:16: F821 undefined name 'nn'
        return nn.Sequential(*layers)
               ^
./pytorch-cifar/resnet18.py:66:17: F821 undefined name 'F'
        x_avg = F.adaptive_avg_pool2d(x, (1, 1))
                ^
./pytorch-cifar/resnet18.py:69:17: F821 undefined name 'F'
        x_max = F.adaptive_max_pool2d(x, (1, 1))
                ^
./pytorch-cifar/resnet18.py:72:13: F821 undefined name 'torch'
        x = torch.cat([x_avg, x_max], dim=-1)
            ^
./iperf/launch.py:109:16: F821 undefined name 'ami_dict_amazon'
    ami_dict = ami_dict_amazon
               ^
./iperf/launch.py:137:9: F821 undefined name 'job'
  print(job.connect_instructions)
        ^
11    E999 SyntaxError: invalid token
90    F821 undefined name 'task'
101
yaroslavvb commented 5 years ago

This project exists to keep history of imagenet/ncluster repos, it's not intended to be functional