Skip to content

Commit

Permalink
port changes
Browse files Browse the repository at this point in the history
  • Loading branch information
jackalcooper committed Aug 2, 2024
1 parent 1329b9d commit 36b96ae
Show file tree
Hide file tree
Showing 29 changed files with 48 additions and 9 deletions.
3 changes: 2 additions & 1 deletion ci/fixed-dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
numpy==1.22.1 ; python_version >= "3.10"
numpy==1.26.4 ; python_version >= "3.12"
numpy==1.22.1 ; python_version >= "3.10" and python_version < "3.12"
numpy==1.21.6 ; python_version >= "3.7" and python_version < "3.10"
1 change: 1 addition & 0 deletions oneflow/api/python/autograd/autograd_function_state.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_AUTOGRAD_AUTOGRAD_FUNCTION_STATE_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>

#include "oneflow/core/framework/op_expr_grad_function.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/caster/size.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_CASTER_SIZE_H_
#include <type_traits>
#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>

#include "oneflow/api/python/framework/size.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/exception/exception.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_COMMON_EXCEPTION_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>

#include "oneflow/core/common/exception.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/memory_format.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FRAMEWORK_MEMORY_FORMAT_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>

#include "oneflow/core/common/memory_format.pb.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/size.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FRAMEWORK_SIZE_H_
#include <type_traits>
#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>
#include "oneflow/core/common/shape.h"

Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/tensor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.

#include <pybind11/pybind11.h>
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/api/python/exception/exception.h"
#include "oneflow/api/python/framework/size.h"
#include "oneflow/api/python/framework/tensortype.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/tensor.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FRAMEWORK_TENSOR_H_

#include <Python.h>
#undef _PyGC_FINALIZED

#include "oneflow/core/framework/tensor.h"

Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/tensor_functions.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ limitations under the License.
*/

#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/api/python/exception/exception.h"
#include "oneflow/api/python/framework/size.h"
#include "oneflow/api/python/framework/tensor_functions_util.h"
Expand Down
3 changes: 2 additions & 1 deletion oneflow/api/python/framework/tensor_functions_util.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
#include <Python.h>
#undef _PyGC_FINALIZED
#include <string>
#include "oneflow/api/python/exception/exception.h"
#include "oneflow/api/python/functional/common.h"
Expand Down Expand Up @@ -71,4 +72,4 @@ PyObject* PyParseArgs(PyObject* args, PyObject* kwargs, const char* func_name,
}

} // namespace one
} // namespace oneflow
} // namespace oneflow
1 change: 1 addition & 0 deletions oneflow/api/python/framework/tensortype.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>
#include "oneflow/api/python/framework/tensor.h"
#include "oneflow/api/python/framework/tensortype.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/tensortype.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FRAMEWORK_TENSORTYPE_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/framework/dtype.h"
#include "oneflow/core/framework/device.h"

Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/framework/typeinfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FRAMEWORK_TYPEINFO_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/common/throw.h"
#include "oneflow/core/framework/dtype.h"

Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/indexing.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FUNCTIONAL_INDEXING_H_

#include <Python.h>
#undef _PyGC_FINALIZED

#include "oneflow/api/python/functional/common.h"
#include "oneflow/core/common/maybe.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/python_arg.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ limitations under the License.

#include <pybind11/pybind11.h>
#include <Python.h>
#undef _PyGC_FINALIZED

#include "oneflow/core/common/throw.h"
#include "oneflow/api/python/functional/value_types.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/python_arg_parser.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FUNCTIONAL_PYTHON_ARG_PARSER_H_

#include <Python.h>
#undef _PyGC_FINALIZED

#include "oneflow/api/python/functional/function_def.h"
#include "oneflow/api/python/functional/python_arg.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/python_return_types.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_FUNCTIONAL_PYTHON_RETURN_TYPES_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include <string>
#include <sstream>
#include <structmember.h>
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/tensor_api.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
#include <Python.h>
#undef _PyGC_FINALIZED
#include <memory>

#include "oneflow/api/python/utils/tensor_utils.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/functional/value_types.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ limitations under the License.
#include <complex>
#include <memory>
#include <Python.h>
#undef _PyGC_FINALIZED

#include "oneflow/core/common/data_type.pb.h"
#include "oneflow/core/common/maybe.h"
Expand Down
1 change: 1 addition & 0 deletions oneflow/api/python/utils/tensor_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
#define ONEFLOW_API_PYTHON_UTILS_TENSOR_UTILS_H_

#include <Python.h>
#undef _PyGC_FINALIZED
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/functional.h>
Expand Down
1 change: 1 addition & 0 deletions oneflow/extension/python/py_compute.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ limitations under the License.
#ifndef ONEFLOW_EXTENSION_PYTHON_PY_COMPUTE_H_
#define ONEFLOW_EXTENSION_PYTHON_PY_COMPUTE_H_
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/framework/framework.h"

namespace oneflow {
Expand Down
1 change: 1 addition & 0 deletions oneflow/extension/python/py_kernel_registry.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ limitations under the License.

#include <string>
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/common/maybe.h"

namespace oneflow {
Expand Down
1 change: 1 addition & 0 deletions oneflow/extension/stack/python/custom_eval_frame.c
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ limitations under the License.

#define PY_SSIZE_T_CLEAN
#include <Python.h>
#undef _PyGC_FINALIZED
#include <frameobject.h>
#include <pystate.h>
// see https://bugs.python.org/issue35886
Expand Down
1 change: 1 addition & 0 deletions oneflow/extension/stack/python/custom_eval_frame.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ extern "C" {
#endif

#include <Python.h>
#undef _PyGC_FINALIZED

#if PY_VERSION_HEX >= 0x03090000
typedef PyObject* (*PyFrameEvalFunc)(struct _ts*, struct _frame*, int);
Expand Down
5 changes: 2 additions & 3 deletions python/oneflow/framework/check_point_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
IO,
BinaryIO,
)
from typing_extensions import TypeAlias
from pathlib import Path
import pickle
import json
Expand Down Expand Up @@ -58,10 +57,10 @@
PROTOCOL_VERSION = 1
ONEFLOW_MAGIC_KEY = "__oneflow__"

MAP_LOCATION: TypeAlias = Optional[
MAP_LOCATION = Optional[
Union[Callable[[Tensor, str], Tensor], flow.device, str, flow.placement]
]
FILE_LIKE: TypeAlias = Union[os.PathLike, BinaryIO, IO[bytes], Path]
FILE_LIKE = Union[os.PathLike, BinaryIO, IO[bytes], Path]


class _opener(object):
Expand Down
14 changes: 10 additions & 4 deletions python/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""

from __future__ import absolute_import

import argparse
Expand Down Expand Up @@ -96,11 +97,16 @@ def has_ext_modules(self):


def get_oneflow_internal_so_path():
import imp

fp, pathname, description = imp.find_module("_oneflow_internal", ["oneflow"])
import importlib

suffixes = importlib.machinery.EXTENSION_SUFFIXES
loader = importlib.machinery.ExtensionFileLoader
lazy_loader = importlib.util.LazyLoader.factory(loader)
finder = importlib.machinery.FileFinder("oneflow", (lazy_loader, suffixes))
spec = finder.find_spec("_oneflow_internal")
pathname = spec.origin
assert os.path.isfile(pathname)
return os.path.relpath(pathname, "oneflow")
return os.path.basename(pathname)


package_data = {"oneflow": [get_oneflow_internal_so_path()] + include_files}
Expand Down
3 changes: 3 additions & 0 deletions tools/functional/generate_dispatch_stateful_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import re
import argparse
Expand Down Expand Up @@ -52,6 +53,7 @@
#define ONEFLOW_API_PYTHON_FUNCTIONAL_GENERATED_DISPATCH_OP_API_H_
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/common/optional.h"
#include "oneflow/core/common/scalar.h"
Expand Down Expand Up @@ -109,6 +111,7 @@
+ """
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/api/python/of_api_registry.h"
#include "oneflow/api/python/functional/common.h"
Expand Down
3 changes: 3 additions & 0 deletions tools/functional/generate_functional_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import re
import argparse
Expand Down Expand Up @@ -101,6 +102,7 @@
+ """
#include <Python.h>
#undef _PyGC_FINALIZED
namespace oneflow {{
namespace one {{
Expand All @@ -117,6 +119,7 @@
+ """
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/api/python/of_api_registry.h"
#include "oneflow/api/python/exception/exception.h"
Expand Down
4 changes: 4 additions & 0 deletions tools/functional/generate_tensor_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import re
import argparse
Expand Down Expand Up @@ -52,6 +53,7 @@
#define ONEFLOW_API_PYTHON_FUNCTIONAL_GENERATED_TENSOR_API_H_
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/core/common/optional.h"
#include "oneflow/core/common/scalar.h"
Expand Down Expand Up @@ -95,6 +97,7 @@
+ """
#include <Python.h>
#undef _PyGC_FINALIZED
namespace oneflow {{
namespace one {{
Expand All @@ -111,6 +114,7 @@
+ """
#include <Python.h>
#undef _PyGC_FINALIZED
#include "oneflow/api/python/of_api_registry.h"
#include "oneflow/api/python/exception/exception.h"
Expand Down

0 comments on commit 36b96ae

Please sign in to comment.