Created
March 31, 2020 04:06
-
-
Save nicolov/09af3c1a8c7dcf971822daf8f2dc9caf to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/third_party/gloo/gloo/transport/tcp/device.cc b/third_party/gloo/gloo/transport/tcp/device.cc | |
index e56c2ad372ef..76caa79c5a8c 100644 | |
--- a/third_party/gloo/gloo/transport/tcp/device.cc | |
+++ b/third_party/gloo/gloo/transport/tcp/device.cc | |
@@ -8,6 +8,7 @@ | |
#include "gloo/transport/tcp/device.h" | |
+#include <array> | |
#include <ifaddrs.h> | |
#include <netdb.h> | |
#include <netinet/in.h> | |
diff --git a/third_party/pytorch/BUILD.bazel b/third_party/pytorch/BUILD.bazel | |
index 3556452d45ae..a76b27324335 100644 | |
--- a/third_party/pytorch/BUILD.bazel | |
+++ b/third_party/pytorch/BUILD.bazel | |
@@ -105,19 +105,35 @@ cc_test( | |
], | |
) | |
+# For some reason, these are only picked up on remote workers if they're present | |
+# in the data of the gen script, but also need to be declared in the genrule | |
+# for $(location). | |
+gen_srcs = [ | |
+ "aten/src/ATen/Declarations.cwrap", | |
+ "aten/src/THCUNN/generic/THCUNN.h", | |
+ "aten/src/ATen/nn.yaml", | |
+ "aten/src/ATen/native/native_functions.yaml", | |
+] + glob(["aten/src/ATen/templates/**"]) | |
+ | |
py_binary( | |
name = "gen", | |
- srcs = ["aten/src/ATen/gen.py"], | |
+ srcs = [ | |
+ "aten/src/ATen/gen.py", | |
+ "aten/src/ATen/cwrap_parser.py", | |
+ "aten/src/ATen/nn_parse.py", | |
+ "aten/src/ATen/common_with_cwrap.py", | |
+ "aten/src/ATen/native_parse.py", | |
+ "aten/src/ATen/preprocess_declarations.py", | |
+ "aten/src/ATen/function_wrapper.py", | |
+ "aten/src/ATen/code_template.py", | |
+ "aten/src/ATen/gen_backend_select_register.py", | |
+ ], | |
+ data = gen_srcs, | |
) | |
genrule( | |
name = "generated_cpp", | |
- srcs = [ | |
- "aten/src/ATen/Declarations.cwrap", | |
- "aten/src/THCUNN/generic/THCUNN.h", | |
- "aten/src/ATen/nn.yaml", | |
- "aten/src/ATen/native/native_functions.yaml", | |
- ] + glob(["aten/src/ATen/templates/**"]), | |
+ srcs = gen_srcs, | |
outs = [ | |
"aten/src/ATen/Declarations.yaml", | |
"aten/src/ATen/CPUType.h", | |
@@ -148,31 +164,6 @@ py_library( | |
imports = ["aten"], | |
) | |
-py_library( | |
- name = "tools_autograd", | |
- srcs = glob(["tools/autograd/*.py"]), | |
- data = glob([ | |
- "tools/autograd/*.yaml", | |
- "tools/autograd/templates/*", | |
- ]), | |
- deps = [":code_template"], | |
-) | |
- | |
-py_library( | |
- name = "tools_jit", | |
- srcs = glob(["tools/jit/*.py"]), | |
- data = glob(["tools/jit/templates/*"]), | |
-) | |
- | |
-py_binary( | |
- name = "generate_code", | |
- srcs = ["tools/setup_helpers/generate_code.py"], | |
- deps = [ | |
- ":tools_autograd", | |
- ":tools_jit", | |
- ], | |
-) | |
- | |
genrule( | |
name = "generated_code", | |
srcs = [ | |
@@ -199,8 +190,8 @@ genrule( | |
"torch/csrc/jit/generated/register_aten_ops_1.cpp", | |
"torch/csrc/jit/generated/register_aten_ops_2.cpp", | |
], | |
- cmd = "$(location :generate_code) --install_dir `dirname $(location torch/csrc/autograd/generated/variable_factories.h)`/../.. --declarations-path $(location aten/src/ATen/Declarations.yaml) --nn-path aten/src", | |
- tools = [":generate_code"], | |
+ cmd = "$(location //tools:generate_code) --install_dir `dirname $(location torch/csrc/autograd/generated/variable_factories.h)`/../.. --declarations-path $(location aten/src/ATen/Declarations.yaml) --nn-path aten/src", | |
+ tools = ["//tools:generate_code"], | |
) | |
exports_files( | |
@@ -627,6 +618,7 @@ intern_build_aten_ops( | |
deps = [ | |
":aten_headers", | |
"@fbgemm", | |
+ "@sleef", | |
], | |
) | |
@@ -673,6 +665,7 @@ cc_library( | |
":torch_headers", | |
"@fbgemm", | |
"@ideep", | |
+ "@sleef", | |
], | |
alwayslink = True, | |
) | |
@@ -2051,18 +2044,23 @@ cc_library( | |
"-Wl,--rpath", | |
"-Wl,/opt/conda/lib", | |
"-L/opt/conda/lib", | |
- "-lpython3.6m", | |
], | |
visibility = ["//visibility:public"], | |
deps = [ | |
":caffe2", | |
":torch_headers", | |
"@local_config_python//:python_headers", | |
+ "@python", | |
"@pybind11", | |
], | |
alwayslink = True, | |
) | |
+alias( | |
+ name = "pytorch", | |
+ actual = "torch", | |
+) | |
+ | |
cc_library( | |
name = "libtorch_headers", | |
hdrs = glob([ | |
diff --git a/third_party/pytorch/aten/src/ATen/code_template.py b/third_party/pytorch/aten/src/ATen/code_template.py | |
index cd0cfdd8cf52..71c854da2609 100644 | |
--- a/third_party/pytorch/aten/src/ATen/code_template.py | |
+++ b/third_party/pytorch/aten/src/ATen/code_template.py | |
@@ -1,3 +1,4 @@ | |
+import os | |
import re | |
# match $identifier or ${identifier} and replace with value in env | |
@@ -26,7 +27,8 @@ class CodeTemplate(object): | |
@staticmethod | |
def from_file(filename): | |
- with open(filename, 'r') as f: | |
+ base_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..") | |
+ with open(os.path.join(base_dir, filename), 'r') as f: | |
return CodeTemplate(f.read(), filename) | |
def __init__(self, pattern, filename=""): | |
diff --git a/third_party/pytorch/aten/src/ATen/common_with_cwrap.py b/third_party/pytorch/aten/src/ATen/common_with_cwrap.py | |
index 668927a3ce1f..a696595522f0 100644 | |
--- a/third_party/pytorch/aten/src/ATen/common_with_cwrap.py | |
+++ b/third_party/pytorch/aten/src/ATen/common_with_cwrap.py | |
@@ -1,6 +1,7 @@ | |
# this code should be common among cwrap and ATen preprocessing | |
# for now, I have put it in one place but right now is copied out of cwrap | |
+import os | |
def parse_arguments(args): | |
new_args = [] | |
@@ -131,7 +132,8 @@ class Argument(object): | |
def parse_header(path): | |
- with open(path, 'r') as f: | |
+ base_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..") | |
+ with open(os.path.join(base_dir, path), 'r') as f: | |
lines = f.read().split('\n') | |
# Remove empty lines and prebackend directives | |
diff --git a/third_party/pytorch/aten/src/ATen/cwrap_parser.py b/third_party/pytorch/aten/src/ATen/cwrap_parser.py | |
index 0aaf9ec8f223..e339fe6bdfd3 100644 | |
--- a/third_party/pytorch/aten/src/ATen/cwrap_parser.py | |
+++ b/third_party/pytorch/aten/src/ATen/cwrap_parser.py | |
@@ -1,4 +1,6 @@ | |
+import os | |
import yaml | |
+ | |
try: | |
# use faster C loader if available | |
from yaml import CLoader as Loader | |
@@ -9,7 +11,8 @@ except ImportError: | |
def parse(filename): | |
- with open(filename, 'r') as file: | |
+ base_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..") | |
+ with open(os.path.join(base_dir, filename), 'r') as file: | |
declaration_lines = [] | |
declarations = [] | |
in_declaration = False | |
diff --git a/third_party/pytorch/aten/src/ATen/native_parse.py b/third_party/pytorch/aten/src/ATen/native_parse.py | |
index 287292163cdf..2c6bdd07d233 100644 | |
--- a/third_party/pytorch/aten/src/ATen/native_parse.py | |
+++ b/third_party/pytorch/aten/src/ATen/native_parse.py | |
@@ -1,4 +1,5 @@ | |
from __future__ import print_function | |
+import os | |
import re | |
import yaml | |
import pprint | |
@@ -366,7 +367,8 @@ def parse_return_arguments(return_decl, inplace, func_decl): | |
def parse_native_yaml(path): | |
- with open(path, 'r') as f: | |
+ base_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..") | |
+ with open(os.path.join(base_dir, path), 'r') as f: | |
return yaml.load(f, Loader=Loader) | |
diff --git a/third_party/pytorch/aten/src/ATen/nn_parse.py b/third_party/pytorch/aten/src/ATen/nn_parse.py | |
index 0037f4343a77..b4bc0368e7ff 100644 | |
--- a/third_party/pytorch/aten/src/ATen/nn_parse.py | |
+++ b/third_party/pytorch/aten/src/ATen/nn_parse.py | |
@@ -1,4 +1,5 @@ | |
import copy | |
+import os | |
import re | |
import common_with_cwrap | |
import yaml | |
@@ -335,7 +336,8 @@ def backward_declaration(base, thnn_functions, backend_types): | |
def parse_nn_yaml(filename): | |
- with open(filename, 'r') as f: | |
+ base_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..") | |
+ with open(os.path.join(base_dir, filename), 'r') as f: | |
return yaml.load(f, Loader=Loader) | |
diff --git a/third_party/pytorch/third_party/gloo.BUILD b/third_party/pytorch/third_party/gloo.BUILD | |
index 66be151e6cd0..df2621df82f6 100644 | |
--- a/third_party/pytorch/third_party/gloo.BUILD | |
+++ b/third_party/pytorch/third_party/gloo.BUILD | |
@@ -1,7 +1,7 @@ | |
load("@rules_cc//cc:defs.bzl", "cc_library") | |
-load("@//tools/rules:cu.bzl", "cu_library") | |
-load("@//third_party:substitution.bzl", "template_rule") | |
-load("@//tools/config:defs.bzl", "if_cuda") | |
+load("@pytorch//tools/rules:cu.bzl", "cu_library") | |
+load("@pytorch//third_party:substitution.bzl", "template_rule") | |
+load("@pytorch//tools/config:defs.bzl", "if_cuda") | |
template_rule( | |
name = "gloo_config_cmake_macros", | |
diff --git a/third_party/pytorch/third_party/mkl-dnn.BUILD b/third_party/pytorch/third_party/mkl-dnn.BUILD | |
index fdb887c9cacc..022c8e297770 100644 | |
--- a/third_party/pytorch/third_party/mkl-dnn.BUILD | |
+++ b/third_party/pytorch/third_party/mkl-dnn.BUILD | |
@@ -1,5 +1,5 @@ | |
load("@rules_cc//cc:defs.bzl", "cc_library") | |
-load("@//third_party:substitution.bzl", "template_rule") | |
+load("@pytorch//third_party:substitution.bzl", "template_rule") | |
template_rule( | |
name = "include_dnnl_version", | |
@@ -19,7 +19,7 @@ template_rule( | |
out = "include/dnnl_config.h", | |
substitutions = { | |
"cmakedefine": "define", | |
- "${DNNL_CPU_THREADING_RUNTIME}": "OMP", | |
+ "${DNNL_CPU_THREADING_RUNTIME}": "SEQ", | |
"${DNNL_CPU_RUNTIME}": "OMP", | |
"${DNNL_GPU_RUNTIME}": "NONE", | |
}, | |
@@ -66,7 +66,7 @@ cc_library( | |
"-fno-strict-overflow", | |
"-fopenmp", | |
] + select({ | |
- "@//tools/config:thread_sanitizer": ["-DMKLDNN_THR=0"], | |
+ "@pytorch//tools/config:thread_sanitizer": ["-DMKLDNN_THR=0"], | |
"//conditions:default": ["-DMKLDNN_THR=2"], | |
}), | |
includes = [ | |
@@ -83,7 +83,7 @@ cc_library( | |
deps = [ | |
"@mkl", | |
] + select({ | |
- "@//tools/config:thread_sanitizer": [], | |
+ "@pytorch//tools/config:thread_sanitizer": [], | |
"//conditions:default": ["@tbb"], | |
}), | |
) | |
diff --git a/third_party/pytorch/third_party/mkl.BUILD b/third_party/pytorch/third_party/mkl.BUILD | |
index bc868b24e83c..c3115f164a66 100644 | |
--- a/third_party/pytorch/third_party/mkl.BUILD | |
+++ b/third_party/pytorch/third_party/mkl.BUILD | |
@@ -13,7 +13,7 @@ cc_library( | |
"libmkl_vml_avx512.so", | |
"libmkl_vml_def.so", | |
] + select({ | |
- "@//tools/config:thread_sanitizer": [], | |
+ "@pytorch//tools/config:thread_sanitizer": [], | |
"//conditions:default": ["libmkl_tbb_thread.so"], | |
}), | |
visibility = ["//visibility:public"], | |
diff --git a/third_party/pytorch/third_party/sleef.BUILD b/third_party/pytorch/third_party/sleef.BUILD | |
index 6b7349e6696c..3ff139febfe5 100644 | |
--- a/third_party/pytorch/third_party/sleef.BUILD | |
+++ b/third_party/pytorch/third_party/sleef.BUILD | |
@@ -1,5 +1,5 @@ | |
load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") | |
-load("@//third_party:sleef.bzl", "sleef_cc_library") | |
+load("@pytorch//third_party:sleef.bzl", "sleef_cc_library") | |
SLEEF_COPTS = [ | |
"-DHAVE_MALLOC_USABLE_SIZE=1", | |
diff --git a/third_party/pytorch/tools/config/BUILD b/third_party/pytorch/tools/config/BUILD | |
index a8f9d0452fce..596e13b6a56b 100644 | |
--- a/third_party/pytorch/tools/config/BUILD | |
+++ b/third_party/pytorch/tools/config/BUILD | |
@@ -7,16 +7,6 @@ config_setting( | |
}, | |
) | |
-# Even when building with --config=cuda, host targets should be built with cuda disabled | |
-# as these targets will run on CI machines that have no GPUs. | |
-selects.config_setting_group( | |
- name = "cuda_enabled_and_capable", | |
- match_all = [ | |
- ":cuda", | |
- "//tools/toolchain:is_cuda_capable", | |
- ], | |
-) | |
- | |
# Configures the system to build with cuda using clang. | |
config_setting( | |
name = "cuda_clang", | |
diff --git a/third_party/pytorch/tools/config/defs.bzl b/third_party/pytorch/tools/config/defs.bzl | |
index 6ddd0e991561..b02e262c7d02 100644 | |
--- a/third_party/pytorch/tools/config/defs.bzl | |
+++ b/third_party/pytorch/tools/config/defs.bzl | |
@@ -8,7 +8,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") | |
def if_cuda(if_true, if_false = []): | |
"""Helper for selecting based on the whether CUDA is configured. """ | |
return selects.with_or({ | |
- "@//tools/config:cuda_enabled_and_capable": if_true, | |
+ # "//tools/config:cuda_enabled_and_capable": if_true, | |
"//conditions:default": if_false, | |
}) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment