diff --git a/python/README.md b/python/README.md index ab39350254..febdf16d45 100644 --- a/python/README.md +++ b/python/README.md @@ -2,6 +2,11 @@ This directory contains the Protobuf library for Python. +For user documentation about how to use Protobuf Python, see +https://protobuf.dev/getting-started/pythontutorial/ + +# Installation + In most cases you should install the library using `pip` or another package manager: @@ -12,10 +17,7 @@ $ pip install protobuf The packages released on https://pypi.org/project/protobuf/#files include both a source distribution and binary wheels. -For user documentation about how to use Protobuf Python, see -https://protobuf.dev/getting-started/pythontutorial/ - -# Building packages from this repo +## Building packages from this repo If for some reason you wish to build the packages directly from this repo, you can use the following Bazel commands: @@ -29,6 +31,15 @@ The binary wheel will build against whatever version of Python is installed on your system. The source package is always the same and does not depend on a local version of Python. +## Building from `setup.py` + +We support building from `setup.py`, but only from a Python source package. +You cannot build from `setup.py` using the GitHub repo or the GitHub source +tarball. + +To build a source package from this repo, see the instructions in the previous +section. + # Implementation backends There are three separate implementations of Python Protobuf. All of them offer diff --git a/python/build_targets.bzl b/python/build_targets.bzl index a1f1767fef..0a20662c3c 100644 --- a/python/build_targets.bzl +++ b/python/build_targets.bzl @@ -461,7 +461,6 @@ def build_targets(name): ":python_src_files", "README.md", "google/__init__.py", - "setup.cfg", ], strip_prefix = "", visibility = ["//python/dist:__pkg__"], @@ -485,8 +484,6 @@ def build_targets(name): "google/protobuf/python_protobuf.h", "internal.bzl", "python_version_test.py", - "setup.cfg", - "setup.py", ], strip_prefix = strip_prefix.from_root(""), visibility = ["//pkg:__pkg__"], diff --git a/python/setup.cfg b/python/setup.cfg deleted file mode 100644 index 2a9acf13da..0000000000 --- a/python/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal = 1 diff --git a/python/setup.py b/python/setup.py deleted file mode 100755 index 329dece306..0000000000 --- a/python/setup.py +++ /dev/null @@ -1,435 +0,0 @@ -#! /usr/bin/env python -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd -# -# See README for usage instructions. - -# pylint:disable=missing-module-docstring -# pylint:disable=g-bad-import-order -import fnmatch -import glob -import os -import pkg_resources -import re -import shutil -import subprocess -import sys -import sysconfig - -# pylint:disable=g-importing-member -# pylint:disable=g-multiple-import - -from setuptools import setup, Extension, find_packages - -from setuptools.command.build_ext import build_ext as _build_ext -from setuptools.command.build_py import build_py as _build_py - -# Find the Protocol Compiler. -if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']): - protoc = os.environ['PROTOC'] -elif os.path.exists('../bazel-bin/protoc'): - protoc = '../bazel-bin/protoc' -elif os.path.exists('../bazel-bin/protoc.exe'): - protoc = '../bazel-bin/protoc.exe' -elif os.path.exists('../protoc'): - protoc = '../protoc' -elif os.path.exists('../protoc.exe'): - protoc = '../protoc.exe' -elif os.path.exists('../vsprojects/Debug/protoc.exe'): - protoc = '../vsprojects/Debug/protoc.exe' -elif os.path.exists('../vsprojects/Release/protoc.exe'): - protoc = '../vsprojects/Release/protoc.exe' -else: - protoc = shutil.which('protoc') - - -def GetVersion(): - """Reads and returns the version from google/protobuf/__init__.py. - - Do not import google.protobuf.__init__ directly, because an installed - protobuf library may be loaded instead. - - Returns: - The version. - """ - - with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file: - exec(version_file.read(), globals()) # pylint:disable=exec-used - return __version__ # pylint:disable=undefined-variable - - -def GenProto(source, require=True): - """Generates a _pb2.py from the given .proto file. - - Does nothing if the output already exists and is newer than the input. - - Args: - source: the .proto file path. - require: if True, exit immediately when a path is not found. - """ - - if not require and not os.path.exists(source): - return - - output = source.replace('.proto', '_pb2.py').replace('../src/', '') - - if (not os.path.exists(output) or - (os.path.exists(source) and - os.path.getmtime(source) > os.path.getmtime(output))): - print('Generating %s...' % output) - - if not os.path.exists(source): - sys.stderr.write("Can't find required file: %s\n" % source) - sys.exit(-1) - - if protoc is None: - sys.stderr.write( - 'protoc is not installed nor found in ../src. Please compile it ' - 'or install the binary package.\n') - sys.exit(-1) - - protoc_command = [protoc, '-I../src', '-I.', '--python_out=.', source] - if subprocess.call(protoc_command) != 0: - sys.exit(-1) - - -def GenerateUnittestProtos(): - """Generates protobuf code for unittests.""" - GenProto('../src/google/protobuf/any_test.proto', False) - GenProto('../src/google/protobuf/map_proto2_unittest.proto', False) - GenProto('../src/google/protobuf/map_unittest.proto', False) - GenProto('../src/google/protobuf/test_messages_proto3.proto', False) - GenProto('../src/google/protobuf/test_messages_proto2.proto', False) - GenProto('../src/google/protobuf/unittest.proto', False) - GenProto('../src/google/protobuf/unittest_custom_options.proto', False) - GenProto('../src/google/protobuf/unittest_import.proto', False) - GenProto('../src/google/protobuf/unittest_import_public.proto', False) - GenProto('../src/google/protobuf/unittest_mset.proto', False) - GenProto('../src/google/protobuf/unittest_mset_wire_format.proto', False) - GenProto('../src/google/protobuf/unittest_no_generic_services.proto', False) - GenProto('../src/google/protobuf/unittest_proto3_arena.proto', False) - GenProto('../src/google/protobuf/unittest_retention.proto', False) - GenProto('../src/google/protobuf/util/json_format.proto', False) - GenProto('../src/google/protobuf/util/json_format_proto3.proto', False) - GenProto('google/protobuf/internal/any_test.proto', False) - GenProto('google/protobuf/internal/descriptor_pool_test1.proto', False) - GenProto('google/protobuf/internal/descriptor_pool_test2.proto', False) - GenProto('google/protobuf/internal/factory_test1.proto', False) - GenProto('google/protobuf/internal/factory_test2.proto', False) - GenProto('google/protobuf/internal/file_options_test.proto', False) - GenProto('google/protobuf/internal/import_test_package/import_public.proto', - False) - GenProto( - 'google/protobuf/internal/import_test_package/import_public_nested.proto', - False) - GenProto('google/protobuf/internal/import_test_package/inner.proto', False) - GenProto('google/protobuf/internal/import_test_package/outer.proto', False) - GenProto('google/protobuf/internal/missing_enum_values.proto', False) - GenProto('google/protobuf/internal/message_set_extensions.proto', False) - GenProto('google/protobuf/internal/more_extensions.proto', False) - GenProto('google/protobuf/internal/more_extensions_dynamic.proto', False) - GenProto('google/protobuf/internal/more_messages.proto', False) - GenProto('google/protobuf/internal/no_package.proto', False) - GenProto('google/protobuf/internal/packed_field_test.proto', False) - GenProto('google/protobuf/internal/test_bad_identifiers.proto', False) - GenProto('google/protobuf/internal/test_proto3_optional.proto', False) - GenProto('google/protobuf/pyext/python.proto', False) - - -class BuildPyCmd(_build_py): - """Custom build_py command for building the protobuf runtime.""" - - def run(self): - # Generate necessary .proto file if it doesn't exist. - GenProto('../src/google/protobuf/descriptor.proto') - GenProto('../src/google/protobuf/compiler/plugin.proto') - GenProto('../src/google/protobuf/any.proto') - GenProto('../src/google/protobuf/api.proto') - GenProto('../src/google/protobuf/duration.proto') - GenProto('../src/google/protobuf/empty.proto') - GenProto('../src/google/protobuf/field_mask.proto') - GenProto('../src/google/protobuf/source_context.proto') - GenProto('../src/google/protobuf/struct.proto') - GenProto('../src/google/protobuf/timestamp.proto') - GenProto('../src/google/protobuf/type.proto') - GenProto('../src/google/protobuf/wrappers.proto') - GenerateUnittestProtos() - - # _build_py is an old-style class, so super() doesn't work. - _build_py.run(self) - - def find_package_modules(self, package, package_dir): - exclude = ( - '*test*', - 'google/protobuf/internal/*_pb2.py', - 'google/protobuf/internal/_parameterized.py', - 'google/protobuf/pyext/python_pb2.py', - ) - modules = _build_py.find_package_modules(self, package, package_dir) - return [(pkg, mod, fil) for (pkg, mod, fil) in modules - if not any(fnmatch.fnmatchcase(fil, pat=pat) for pat in exclude)] - - -class BuildExtCmd(_build_ext): - """Command class for building the protobuf Python extension.""" - - def get_ext_filename(self, ext_name): - # since python3.5, python extensions' shared libraries use a suffix that - # corresponds to the value of sysconfig.get_config_var('EXT_SUFFIX') and - # contains info about the architecture the library targets. E.g. on x64 - # linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so" When - # crosscompiling python wheels, we need to be able to override this - # suffix so that the resulting file name matches the target architecture - # and we end up with a well-formed wheel. - filename = _build_ext.get_ext_filename(self, ext_name) - orig_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') - new_ext_suffix = os.getenv('PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX') - if new_ext_suffix and filename.endswith(orig_ext_suffix): - filename = filename[:-len(orig_ext_suffix)] + new_ext_suffix - return filename - - -class TestConformanceCmd(_build_py): - target = '//python:conformance_test' - - def run(self): - # Python 2.6 dodges these extra failures. - os.environ['CONFORMANCE_PYTHON_EXTRA_FAILURES'] = ( - '--failure_list failure_list_python-post26.txt') - cmd = 'bazel test %s' % (TestConformanceCmd.target,) - subprocess.check_call(cmd, shell=True) - - -def GetOptionFromArgv(option_str): - if option_str in sys.argv: - sys.argv.remove(option_str) - return True - return False - - -def _GetFlagValues(flag_long, flag_short): - """Searches sys.argv for setuptools-style flags and yields values.""" - - expect_value = flag_long.endswith('=') - flag_res = [re.compile(r'--?%s(=(.*))?' % - (flag_long[:-1] if expect_value else flag_long))] - if flag_short: - flag_res.append(re.compile(r'-%s(.*)?' % (flag_short,))) - - flag_match = None - for arg in sys.argv: - # If the last arg was like '-O', check if this is the library we want. - if flag_match is not None: - yield arg - flag_match = None - continue - - for flag_re in flag_res: - m = flag_re.match(arg) - if m is None: - continue - if not expect_value: - yield arg - continue - groups = m.groups() - # Check for matches: - # --long-name=foo => ('=foo', 'foo') - # -Xfoo => ('foo') - # N.B.: if the flag is like '--long-name=', then there is a value - # (the empty string). - if groups[0] or groups[-1]: - yield groups[-1] - continue - flag_match = m - - return False - - -def HasStaticLibprotobufOpt(): - """Returns true if there is a --link-objects arg for libprotobuf.""" - - lib_re = re.compile(r'(.*[/\\])?(lib)?protobuf([.]pic)?[.](a|lib)') - for value in _GetFlagValues('link-objects=', 'O'): - if lib_re.match(value): - return True - return False - - -def HasLibraryDirsOpt(): - """Returns true if there is a --library-dirs arg.""" - return any(_GetFlagValues('library-dirs=', 'L')) - - -if __name__ == '__main__': - ext_module_list = [] - warnings_as_errors = '--warnings_as_errors' - if GetOptionFromArgv('--cpp_implementation'): - # Link libprotobuf.a and libprotobuf-lite.a statically with the - # extension. Note that those libraries have to be compiled with - # -fPIC for this to work. - compile_static_ext = HasStaticLibprotobufOpt() - if GetOptionFromArgv('--compile_static_extension'): - # FUTURE: add a warning and deprecate --compile_static_extension. - compile_static_ext = True - extra_objects = None - if compile_static_ext: - libraries = None - library_dirs = None - if not HasStaticLibprotobufOpt(): - if os.path.exists('../bazel-bin/src/google/protobuf/libprotobuf.a'): - extra_objects = ['../bazel-bin/src/google/protobuf/libprotobuf.a'] - else: - extra_objects = ['../libprotobuf.a'] - extra_objects += list( - glob.iglob('../third_party/utf8_range/*.a')) - # Repeat all of these enough times to eliminate order-dependence. - extra_objects += list( - glob.iglob('../third_party/abseil-cpp/absl/**/*.a')) - extra_objects += list( - glob.iglob('../third_party/abseil-cpp/absl/**/*.a')) - extra_objects += list( - glob.iglob('../third_party/abseil-cpp/absl/**/*.a')) - else: - libraries = ['protobuf'] - if HasLibraryDirsOpt(): - library_dirs = None - elif os.path.exists('../bazel-bin/src/google/protobuf/libprotobuf.a'): - library_dirs = ['../bazel-bin/src/google/protobuf'] - else: - library_dirs = ['..'] - - TestConformanceCmd.target = ('//python:conformance_test_cpp ' - '--define=use_fast_cpp_protos=true') - - extra_compile_args = [] - - message_extra_link_args = None - api_implementation_link_args = None - if 'darwin' in sys.platform: - if sys.version_info[0] == 2: - message_init_symbol = 'init_message' - api_implementation_init_symbol = 'init_api_implementation' - else: - message_init_symbol = 'PyInit__message' - api_implementation_init_symbol = 'PyInit__api_implementation' - message_extra_link_args = [ - '-Wl,-exported_symbol,_%s' % message_init_symbol - ] - api_implementation_link_args = [ - '-Wl,-exported_symbol,_%s' % api_implementation_init_symbol - ] - - if sys.platform != 'win32': - extra_compile_args.append('-Wno-write-strings') - extra_compile_args.append('-Wno-invalid-offsetof') - extra_compile_args.append('-Wno-sign-compare') - extra_compile_args.append('-Wno-unused-variable') - extra_compile_args.append('-std=c++14') - - if sys.platform == 'darwin': - extra_compile_args.append('-Wno-shorten-64-to-32') - extra_compile_args.append('-Wno-deprecated-register') - - # https://developer.apple.com/documentation/xcode_release_notes/xcode_10_release_notes - # C++ projects must now migrate to libc++ and are recommended to set a - # deployment target of macOS 10.9 or later, or iOS 7 or later. - if sys.platform == 'darwin': - mac_target = str(sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')) - if mac_target and (pkg_resources.parse_version(mac_target) < - pkg_resources.parse_version('10.9.0')): - os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9' - os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( - r'macosx-[0-9]+\.[0-9]+-(.+)', - r'macosx-10.9-\1', - sysconfig.get_platform(), - ) - - # https://github.com/Theano/Theano/issues/4926 - if sys.platform == 'win32': - extra_compile_args.append('-D_hypot=hypot') - - # https://github.com/tpaviot/pythonocc-core/issues/48 - if sys.platform == 'win32' and '64 bit' in sys.version: - extra_compile_args.append('-DMS_WIN64') - - # MSVS default is dymanic - if sys.platform == 'win32': - extra_compile_args.append('/MT') - - if 'clang' in os.popen('$CC --version 2> /dev/null').read(): - extra_compile_args.append('-Wno-shorten-64-to-32') - - if warnings_as_errors in sys.argv: - extra_compile_args.append('-Werror') - sys.argv.remove(warnings_as_errors) - - # C++ implementation extension - ext_module_list.extend([ - Extension( - 'google.protobuf.pyext._message', - glob.glob('google/protobuf/pyext/*.cc'), - include_dirs=['.', '../src', '../third_party/abseil-cpp'], - libraries=libraries, - extra_objects=extra_objects, - extra_link_args=message_extra_link_args, - library_dirs=library_dirs, - extra_compile_args=extra_compile_args, - ), - Extension( - 'google.protobuf.internal._api_implementation', - glob.glob('google/protobuf/internal/api_implementation.cc'), - extra_compile_args=(extra_compile_args + - ['-DPYTHON_PROTO2_CPP_IMPL_V2']), - extra_link_args=api_implementation_link_args, - ), - ]) - os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' - - install_requires = [] - - setup( - name='protobuf', - version=GetVersion(), - description='Protocol Buffers', - download_url='https://github.com/protocolbuffers/protobuf/releases', - long_description="Protocol Buffers are Google's data interchange format", - url='https://developers.google.com/protocol-buffers/', - project_urls={ - 'Source': 'https://github.com/protocolbuffers/protobuf', - }, - maintainer='protobuf@googlegroups.com', - maintainer_email='protobuf@googlegroups.com', - license='BSD-3-Clause', - classifiers=[ - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - # LINT.IfChange - # Remove importlib fallback path when we drop Python 3.8 support. - 'Programming Language :: Python :: 3.8', - # LINT.ThenChange(//depot/google3/google/protobuf/internal/test_util.py) - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - ], - namespace_packages=['google'], - packages=find_packages( - exclude=[ - 'import_test_package', - 'protobuf_distutils', - ], - ), - test_suite='google.protobuf.internal', - cmdclass={ - 'build_py': BuildPyCmd, - 'build_ext': BuildExtCmd, - 'test_conformance': TestConformanceCmd, - }, - install_requires=install_requires, - ext_modules=ext_module_list, - python_requires='>=3.8', - )