From 859ad8122d75e8ef4d29736a926f4643e8e23c64 Mon Sep 17 00:00:00 2001 From: Bartek Ciszkowski Date: Tue, 27 Oct 2026 15:38:05 -0300 Subject: [PATCH 1/2] TIKA-5009: Make projectId optional in the GCS plugin The GCS fetcher, emitter and pipes iterator all fail to build without a projectId, but none of their calls need one. They read, write and list objects in a specified bucket. The java client also does not require projectId, and a comment within it states "Project ID is only required for creating buckets, so we don't requite it for creating the service.": https://github.com/googleapis/google-cloud-java/blob/123f97c3f843dc7a3ff7b4ef0596c1425502a275/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java#L326 The docs also state projectId picks the project billed for API calls, but storage access is billed to the project that owns the bucket; this has been corrected. --- .../modules/ROOT/pages/pipes/plugins/gcs.adoc | 14 ++-- .../tika/pipes/emitter/gcs/GCSEmitter.java | 9 +-- .../pipes/emitter/gcs/GCSEmitterConfig.java | 3 - .../tika/pipes/fetcher/gcs/GCSFetcher.java | 13 ++-- .../pipes/iterator/gcs/GCSPipesIterator.java | 14 ++-- .../tika/pipes/gcs/GCSProjectIdTest.java | 64 +++++++++++++++++++ 6 files changed, 88 insertions(+), 29 deletions(-) create mode 100644 tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java diff --git a/docs/modules/ROOT/pages/pipes/plugins/gcs.adoc b/docs/modules/ROOT/pages/pipes/plugins/gcs.adoc index dbfcfa8259d..e32ed8386df 100644 --- a/docs/modules/ROOT/pages/pipes/plugins/gcs.adoc +++ b/docs/modules/ROOT/pages/pipes/plugins/gcs.adoc @@ -47,7 +47,7 @@ The GCS plugin relies on Google's Application Default Credentials chain — ther * Setting the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of a service-account JSON key. * Running `gcloud auth application-default login` for local development. -The `projectId` field in each component selects which GCP project to bill the API calls against; the service account or user must have storage access to the named bucket. +`projectId` is optional. Reading, writing and listing objects needs no project, and storage access is billed to the project that owns the bucket. When unset, the client resolves a project from the environment (`GOOGLE_CLOUD_PROJECT`, the credentials file, the gcloud config or the GCE metadata server). The service account or user must have storage access to the named bucket. [#gcs-fetcher] == GCS Fetcher (`gcs-fetcher`) @@ -66,8 +66,8 @@ include::example$pipes-gcs-fetcher.json[] |Field |Default |Description |`projectId` -|_required_ -|GCP project ID for billing/authentication. +|_none_ +|Optional GCP project ID. See <>. |`bucket` |_required_ @@ -99,8 +99,8 @@ include::example$pipes-gcs-emitter.json[] |Field |Default |Description |`projectId` -|_required_ -|GCP project ID (validated non-blank). +|_none_ +|Optional GCP project ID. See <>. |`bucket` |_required_ @@ -138,8 +138,8 @@ In addition to the required `fetcherId` / `emitterId` (see xref:pipes/iterators. |GCS bucket to enumerate. |`projectId` -|`""` -|GCP project ID for the listing API call. +|_none_ +|Optional GCP project ID. See <>. |`prefix` |`""` diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java index e3a9080e5d4..e0bf0b09098 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java @@ -82,10 +82,11 @@ private GCSEmitter(ExtensionConfig extensionConfig, GCSEmitterConfig config, Sto } private static Storage buildStorage(GCSEmitterConfig config) { - return StorageOptions.newBuilder() - .setProjectId(config.projectId()) - .build() - .getService(); + StorageOptions.Builder builder = StorageOptions.newBuilder(); + if (!StringUtils.isBlank(config.projectId())) { + builder.setProjectId(config.projectId()); + } + return builder.build().getService(); } @Override diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterConfig.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterConfig.java index a7408ba4b32..15a3264a557 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterConfig.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterConfig.java @@ -34,9 +34,6 @@ public static GCSEmitterConfig load(final String json) } public void validate() throws TikaConfigException { - if (projectId == null || projectId.isBlank()) { - throw new TikaConfigException("'projectId' must not be empty"); - } if (bucket == null || bucket.isBlank()) { throw new TikaConfigException("'bucket' must not be empty"); } diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java index 1f45615b411..82fc436d070 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java @@ -38,9 +38,10 @@ import org.apache.tika.pipes.fetcher.gcs.config.GCSFetcherConfig; import org.apache.tika.plugins.AbstractTikaExtension; import org.apache.tika.plugins.ExtensionConfig; +import org.apache.tika.utils.StringUtils; /** - * Fetches files from google cloud storage. Must set projectId and bucket via the config. + * Fetches files from google cloud storage. Must set bucket via the config; projectId is optional. */ public class GCSFetcher extends AbstractTikaExtension implements Fetcher { @@ -64,12 +65,12 @@ public static GCSFetcher build(ExtensionConfig extensionConfig) throws IOExcepti private void initialize() throws TikaConfigException { ConfigValidator.mustNotBeEmpty("bucket", config.getBucket()); - ConfigValidator.mustNotBeEmpty("projectId", config.getProjectId()); - storage = StorageOptions.newBuilder() - .setProjectId(config.getProjectId()) - .build() - .getService(); + StorageOptions.Builder builder = StorageOptions.newBuilder(); + if (!StringUtils.isBlank(config.getProjectId())) { + builder.setProjectId(config.getProjectId()); + } + storage = builder.build().getService(); } @Override diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/iterator/gcs/GCSPipesIterator.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/iterator/gcs/GCSPipesIterator.java index 8fb6df1547f..45804e3c896 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/iterator/gcs/GCSPipesIterator.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/iterator/gcs/GCSPipesIterator.java @@ -50,16 +50,12 @@ private GCSPipesIterator(GCSPipesIteratorConfig config, ExtensionConfig extensio if (StringUtils.isBlank(config.getBucket())) { throw new TikaConfigException("bucket must not be empty"); } - if (StringUtils.isBlank(config.getProjectId())) { - throw new TikaConfigException("projectId must not be empty"); - } - // Initialize the GCS client - this.storage = StorageOptions - .newBuilder() - .setProjectId(config.getProjectId()) - .build() - .getService(); + StorageOptions.Builder builder = StorageOptions.newBuilder(); + if (!StringUtils.isBlank(config.getProjectId())) { + builder.setProjectId(config.getProjectId()); + } + this.storage = builder.build().getService(); } public static GCSPipesIterator build(ExtensionConfig extensionConfig) throws IOException, TikaConfigException { diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java new file mode 100644 index 00000000000..55fa2b65dd9 --- /dev/null +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.gcs; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.exception.TikaConfigException; +import org.apache.tika.pipes.emitter.gcs.GCSEmitter; +import org.apache.tika.pipes.fetcher.gcs.GCSFetcher; +import org.apache.tika.pipes.iterator.gcs.GCSPipesIterator; +import org.apache.tika.plugins.ExtensionConfig; + +/** + * projectId is optional: reading, writing and listing objects need no project, so each + * component builds without one. Building makes no request, so these need no credentials. + */ +public class GCSProjectIdTest { + + private static final String BUCKET_ONLY = "{\"bucket\": \"bucket\"}"; + + @Test + public void testFetcherWithoutProjectId() throws Exception { + assertNotNull(GCSFetcher.build(new ExtensionConfig("gcs", "gcs-fetcher", BUCKET_ONLY))); + } + + @Test + public void testFetcherWithProjectId() throws Exception { + String json = "{\"bucket\": \"bucket\", \"projectId\": \"project\"}"; + assertNotNull(GCSFetcher.build(new ExtensionConfig("gcs", "gcs-fetcher", json))); + } + + @Test + public void testFetcherRequiresBucket() { + assertThrows(TikaConfigException.class, + () -> GCSFetcher.build(new ExtensionConfig("gcs", "gcs-fetcher", "{}"))); + } + + @Test + public void testEmitterWithoutProjectId() throws Exception { + assertNotNull(GCSEmitter.build(new ExtensionConfig("gcs", "gcs-emitter", BUCKET_ONLY))); + } + + @Test + public void testIteratorWithoutProjectId() throws Exception { + assertNotNull(GCSPipesIterator.build(new ExtensionConfig("gcs", "gcs-pipes-iterator", BUCKET_ONLY))); + } +} From c603717e88e71c249d3c266b111c2a6ee8c079ab Mon Sep 17 00:00:00 2001 From: Bartek Ciszkowski Date: Tue, 27 Oct 2026 21:07:20 -0300 Subject: [PATCH 2/2] fixup! TIKA-5009: Make projectId optional in the GCS plugin --- CHANGES.txt | 4 +++ .../tika/pipes/gcs/ConfigExamplesTest.java | 4 ++- .../tika/pipes/gcs/GCSProjectIdTest.java | 26 +++++++++++++++++-- .../config-examples/gcs-fetcher.json | 1 - 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 9826420af33..5eb45d84ce9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,10 @@ Release 4.2.1 - unreleased for EKS IAM roles for service accounts (IRSA); the plugin now bundles the AWS STS module it needs (TIKA-5006). + * The GCS fetcher, emitter and pipes iterator no longer require projectId. When it is + unset, the client resolves a project from the environment, as it does for its + credentials (TIKA-5009). + OTHER CHANGES * PEM-armored CMS/PKCS7 (-----BEGIN PKCS7----- or -----BEGIN CMS-----) is parsed like diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/ConfigExamplesTest.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/ConfigExamplesTest.java index 52ce0a1fe56..eab655c387f 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/ConfigExamplesTest.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/ConfigExamplesTest.java @@ -17,6 +17,7 @@ package org.apache.tika.pipes.gcs; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import com.fasterxml.jackson.databind.JsonNode; import org.junit.jupiter.api.Test; @@ -41,7 +42,8 @@ public void testGCSFetcherConfig() throws Exception { JsonNode inner = innerComponent(readExample("gcs-fetcher.json"), "fetchers", "gcsf", "gcs-fetcher"); GCSFetcherConfig config = GCSFetcherConfig.load(inner.toString()); - assertEquals("my-gcp-project", config.getProjectId()); + // The fetcher example leaves projectId out to show it is optional. + assertNull(config.getProjectId()); assertEquals("my-tika-input", config.getBucket()); } diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java index 55fa2b65dd9..cb7ba0c1401 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java @@ -34,6 +34,7 @@ public class GCSProjectIdTest { private static final String BUCKET_ONLY = "{\"bucket\": \"bucket\"}"; + private static final String WITH_PROJECT = "{\"bucket\": \"bucket\", \"projectId\": \"project\"}"; @Test public void testFetcherWithoutProjectId() throws Exception { @@ -42,8 +43,7 @@ public void testFetcherWithoutProjectId() throws Exception { @Test public void testFetcherWithProjectId() throws Exception { - String json = "{\"bucket\": \"bucket\", \"projectId\": \"project\"}"; - assertNotNull(GCSFetcher.build(new ExtensionConfig("gcs", "gcs-fetcher", json))); + assertNotNull(GCSFetcher.build(new ExtensionConfig("gcs", "gcs-fetcher", WITH_PROJECT))); } @Test @@ -57,8 +57,30 @@ public void testEmitterWithoutProjectId() throws Exception { assertNotNull(GCSEmitter.build(new ExtensionConfig("gcs", "gcs-emitter", BUCKET_ONLY))); } + @Test + public void testEmitterWithProjectId() throws Exception { + assertNotNull(GCSEmitter.build(new ExtensionConfig("gcs", "gcs-emitter", WITH_PROJECT))); + } + + @Test + public void testEmitterRequiresBucket() { + assertThrows(TikaConfigException.class, + () -> GCSEmitter.build(new ExtensionConfig("gcs", "gcs-emitter", "{}"))); + } + @Test public void testIteratorWithoutProjectId() throws Exception { assertNotNull(GCSPipesIterator.build(new ExtensionConfig("gcs", "gcs-pipes-iterator", BUCKET_ONLY))); } + + @Test + public void testIteratorWithProjectId() throws Exception { + assertNotNull(GCSPipesIterator.build(new ExtensionConfig("gcs", "gcs-pipes-iterator", WITH_PROJECT))); + } + + @Test + public void testIteratorRequiresBucket() { + assertThrows(TikaConfigException.class, + () -> GCSPipesIterator.build(new ExtensionConfig("gcs", "gcs-pipes-iterator", "{}"))); + } } diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/resources/config-examples/gcs-fetcher.json b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/resources/config-examples/gcs-fetcher.json index 89ab85eed3b..177445ac285 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/resources/config-examples/gcs-fetcher.json +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/resources/config-examples/gcs-fetcher.json @@ -2,7 +2,6 @@ "fetchers": { "gcsf": { "gcs-fetcher": { - "projectId": "my-gcp-project", "bucket": "my-tika-input", "extractUserMetadata": true, "spoolToTemp": true