Merge pull request #12 from benchsci/update-gazelle

Update gazelle to 0.34.0

.bazelrc

@@ -0,0 +1,4 @@
+# Print results from tests
+build --test_output=all
+build --test_summary=terse
+build --incompatible_default_to_explicit_init_py

.bazelversion

@@ -0,0 +1 @@
+6.4.0

.github/workflows/ci.yaml

@@ -32,4 +32,4 @@ jobs:
               env:
                   # Bazelisk will download bazel to here
                   XDG_CACHE_HOME: ~/.cache/bazel-repo
-              run: bazel test //...
+              run: bazel test //... --jobs 2 # limit number of jobs to prevent broken pipe error

WORKSPACE

@@ -16,28 +16,21 @@ workspace(name = "com_github_benchsci_rules_python_gazelle")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-http_archive(
-    name = "rules_python",
-    sha256 = "a30abdfc7126d497a7698c29c46ea9901c6392d6ed315171a6df5ce433aa4502",
-    strip_prefix = "rules_python-0.6.0",
-    url = "https://github.com/bazelbuild/rules_python/archive/0.6.0.tar.gz",
-)
-
 http_archive(
     name = "io_bazel_rules_go",
-    sha256 = "278b7ff5a826f3dc10f04feaf0b70d48b68748ccd512d7f98bf442077f043fe3",
+    sha256 = "91585017debb61982f7054c9688857a2ad1fd823fc3f9cb05048b0025c47d023",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip",
     ],
 )
 
 http_archive(
     name = "bazel_gazelle",
-    sha256 = "d3fa66a39028e97d76f9e2db8f1b0c11c099e8e01bf363a923074784e451f809",
+    sha256 = "b7387f72efb59f876e4daae42f1d3912d0d45563eac7cb23d1de0b094ab588cf",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.34.0/bazel-gazelle-v0.34.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.34.0/bazel-gazelle-v0.34.0.tar.gz",
     ],
 )
 
@@ -53,3 +46,20 @@ go_rules_dependencies()
 go_register_toolchains(version = "1.20.5")
 
 gazelle_dependencies()
+
+http_archive(
+    name = "rules_python",
+    sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b",
+    strip_prefix = "rules_python-0.26.0",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz",
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
+
+# Add python 3.11 toolchain
+python_register_toolchains(
+    name = "python_3_11",
+    python_version = "3.11",
+)
+
+py_repositories()

gazelle/BUILD.bazel

@@ -10,11 +10,13 @@ go_library(
         "generate.go",
         "kinds.go",
         "language.go",
+        "lifecycle.go",
         "parser.go",
         "resolve.go",
         "std_modules.go",
         "target.go",
     ],
+    embedsrcs = [":helper.zip"],  # keep
     importpath = "github.com/benchsci/rules_python_gazelle/gazelle",
     visibility = ["//visibility:public"],
     deps = [
@@ -50,6 +52,25 @@ py_binary(
     visibility = ["//visibility:public"],
 )
 
+py_binary(
+    name = "helper",
+    srcs = [
+        "__main__.py",
+        "parse.py",
+        "std_modules.py",
+    ],
+    # This is to make sure that the current directory is added to PYTHONPATH
+    imports = ["."],
+    main = "__main__.py",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "helper.zip",
+    srcs = [":helper"],
+    output_group = "python_zip_file",
+)
+
 TEST_DEPS = [
     "@bazel_gazelle//testtools:go_default_library",
     "@com_github_emirpasic_gods//lists/singlylinkedlist",

gazelle/__main__.py

@@ -0,0 +1,32 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# parse.py is a long-living program that communicates over STDIN and STDOUT.
+# STDIN receives parse requests, one per line. It outputs the parsed modules and
+# comments from all the files from each request.
+
+import sys
+
+import parse
+import std_modules
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        sys.exit("Please provide subcommand, either print or std_modules")
+    if sys.argv[1] == "parse":
+        sys.exit(parse.main(sys.stdin, sys.stdout))
+    elif sys.argv[1] == "std_modules":
+        sys.exit(std_modules.main(sys.stdin, sys.stdout))
+    else:
+        sys.exit("Unknown subcommand: " + sys.argv[1])

gazelle/language.go

@@ -9,6 +9,7 @@ import (
 type Python struct {
 	Configurer
 	Resolver
+	LifeCycleManager
 }
 
 // NewLanguage initializes a new Python that satisfies the language.Language

gazelle/lifecycle.go

@@ -0,0 +1,63 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package python
+
+import (
+	"context"
+	_ "embed"
+	"github.com/bazelbuild/bazel-gazelle/language"
+	"log"
+	"os"
+)
+
+var (
+	//go:embed helper.zip
+	helperZip  []byte
+	helperPath string
+)
+
+type LifeCycleManager struct {
+	language.BaseLifecycleManager
+	pyzFilePath string
+}
+
+func (l *LifeCycleManager) Before(ctx context.Context) {
+	helperPath = os.Getenv("GAZELLE_PYTHON_HELPER")
+	if helperPath == "" {
+		pyzFile, err := os.CreateTemp("", "python_zip_")
+		if err != nil {
+			log.Fatalf("failed to write parser zip: %v", err)
+		}
+		defer pyzFile.Close()
+		helperPath = pyzFile.Name()
+		l.pyzFilePath = helperPath
+		if _, err := pyzFile.Write(helperZip); err != nil {
+			log.Fatalf("cannot write %q: %v", helperPath, err)
+		}
+	}
+	startParserProcess(ctx)
+	startStdModuleProcess(ctx)
+}
+
+func (l *LifeCycleManager) DoneGeneratingRules() {
+	shutdownParserProcess()
+}
+
+func (l *LifeCycleManager) AfterResolvingDeps(ctx context.Context) {
+	shutdownStdModuleProcess()
+	if l.pyzFilePath != "" {
+		os.Remove(l.pyzFilePath)
+	}
+}

gazelle/modules_mapping/BUILD.bazel

@@ -3,6 +3,7 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_test")
 py_binary(
     name = "generator",
     srcs = ["generator.py"],
+    imports = ["."],
     main = "generator.py",
     visibility = ["//visibility:public"],
 )
@@ -11,5 +12,13 @@ py_test(
     name = "test_generator",
     srcs = ["test_generator.py"],
     data = glob(["testdata/**"]),
+    imports = ["."],
     main = "test_generator.py",
+    deps = [":generator"],
+)
+
+filegroup(
+    name = "distribution",
+    srcs = glob(["**"]),
+    visibility = ["//:__pkg__"],
 )

gazelle/modules_mapping/test_generator.py

@@ -1,7 +1,8 @@
 import pathlib
-from generator import Generator
 import unittest
 
+from generator import Generator
+
 
 class GeneratorTest(unittest.TestCase):
     def test_generator(self):
@@ -28,7 +29,10 @@ def test_stub_generator(self):
         gen = Generator(None, None)
         mapping = gen.dig_wheel(whl)
         self.assertLessEqual(
-            {"django_types": "django_types",}.items(), mapping.items(),
+            {
+                "django_types": "django_types",
+            }.items(),
+            mapping.items(),
         )
 
 

gazelle/parse.py

@@ -9,6 +9,7 @@
 import re
 import sys
 from io import BytesIO
+from multiprocessing import cpu_count
 from tokenize import COMMENT, tokenize
 
 
@@ -88,7 +89,9 @@ def parse(repo_root, rel_package_path, filename):
 
 
 def main(stdin, stdout):
-    with concurrent.futures.ProcessPoolExecutor() as executor:
+    with concurrent.futures.ProcessPoolExecutor(
+        max_workers=max(cpu_count() - 1, 1)
+    ) as executor:
         for parse_request in stdin:
             parse_request = json.loads(parse_request)
             repo_root = parse_request["repo_root"]

gazelle/parser.go

@@ -3,6 +3,7 @@ package python
 import (
 	"bufio"
 	"context"
+	_ "embed"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,58 +13,52 @@ import (
 	"sort"
 	"strings"
 	"sync"
-	"time"
 
-	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/emirpasic/gods/sets/treeset"
 	godsutils "github.com/emirpasic/gods/utils"
 )
 
 var (
-	parserStdin  io.Writer
+	parserCmd    *exec.Cmd
+	parserStdin  io.WriteCloser
 	parserStdout io.Reader
 	parserMutex  sync.Mutex
 )
 
-func init() {
-	parseScriptRunfile, err := bazel.Runfile("gazelle/parse")
-	if err != nil {
-		log.Printf("failed to initialize parser: %v\n", err)
-		os.Exit(1)
-	}
-
-	ctx := context.Background()
-	ctx, parserCancel := context.WithTimeout(ctx, time.Minute*5)
-	cmd := exec.CommandContext(ctx, parseScriptRunfile)
-
-	cmd.Stderr = os.Stderr
+func startParserProcess(ctx context.Context) {
+	// due to #691, we need a system interpreter to boostrap, part of which is
+	// to locate the hermetic interpreter.
+	parserCmd = exec.CommandContext(ctx, "python3", helperPath, "parse")
+	parserCmd.Stderr = os.Stderr
 
-	stdin, err := cmd.StdinPipe()
+	stdin, err := parserCmd.StdinPipe()
 	if err != nil {
 		log.Printf("failed to initialize parser: %v\n", err)
 		os.Exit(1)
 	}
 	parserStdin = stdin
 
-	stdout, err := cmd.StdoutPipe()
+	stdout, err := parserCmd.StdoutPipe()
 	if err != nil {
 		log.Printf("failed to initialize parser: %v\n", err)
 		os.Exit(1)
 	}
 	parserStdout = stdout
 
-	if err := cmd.Start(); err != nil {
+	if err := parserCmd.Start(); err != nil {
 		log.Printf("failed to initialize parser: %v\n", err)
 		os.Exit(1)
 	}
+}
 
-	go func() {
-		defer parserCancel()
-		if err := cmd.Wait(); err != nil {
-			log.Printf("failed to wait for parser: %v\n", err)
-			os.Exit(1)
-		}
-	}()
+func shutdownParserProcess() {
+	if err := parserStdin.Close(); err != nil {
+		fmt.Fprintf(os.Stderr, "error closing parser: %v", err)
+	}
+
+	if err := parserCmd.Wait(); err != nil {
+		log.Printf("failed to wait for parser: %v\n", err)
+	}
 }
 
 // python3Parser implements a parser for Python files that extracts the modules