diff --git a/CHANGES.txt b/CHANGES.txt index bef8eef13b6..2500a895946 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -22,6 +22,9 @@ Release 4.3.0 - ??? for EKS IAM roles for service accounts (IRSA); the plugin now bundles the AWS STS module it needs (TIKA-5006). + * The GCS fetcher, emitter and pipes iterator no longer require projectId. When it is + unset, the client resolves a project from the environment, as it does for its + credentials (TIKA-5009). * The Kafka emitter reports success only once Kafka has acknowledged every record (per acks); a rejected or timed-out record fails the emit instead of being lost silently, and the producer is flushed and closed on shutdown. (TIKA-5008). diff --git a/docs/modules/ROOT/pages/pipes/plugins/gcs.adoc b/docs/modules/ROOT/pages/pipes/plugins/gcs.adoc index dbfcfa8259d..e32ed8386df 100644 --- a/docs/modules/ROOT/pages/pipes/plugins/gcs.adoc +++ b/docs/modules/ROOT/pages/pipes/plugins/gcs.adoc @@ -47,7 +47,7 @@ The GCS plugin relies on Google's Application Default Credentials chain — ther * Setting the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of a service-account JSON key. * Running `gcloud auth application-default login` for local development. -The `projectId` field in each component selects which GCP project to bill the API calls against; the service account or user must have storage access to the named bucket. +`projectId` is optional. Reading, writing and listing objects needs no project, and storage access is billed to the project that owns the bucket. When unset, the client resolves a project from the environment (`GOOGLE_CLOUD_PROJECT`, the credentials file, the gcloud config or the GCE metadata server). The service account or user must have storage access to the named bucket. [#gcs-fetcher] == GCS Fetcher (`gcs-fetcher`) @@ -66,8 +66,8 @@ include::example$pipes-gcs-fetcher.json[] |Field |Default |Description |`projectId` -|_required_ -|GCP project ID for billing/authentication. +|_none_ +|Optional GCP project ID. See <>. |`bucket` |_required_ @@ -99,8 +99,8 @@ include::example$pipes-gcs-emitter.json[] |Field |Default |Description |`projectId` -|_required_ -|GCP project ID (validated non-blank). +|_none_ +|Optional GCP project ID. See <>. |`bucket` |_required_ @@ -138,8 +138,8 @@ In addition to the required `fetcherId` / `emitterId` (see xref:pipes/iterators. |GCS bucket to enumerate. |`projectId` -|`""` -|GCP project ID for the listing API call. +|_none_ +|Optional GCP project ID. See <>. |`prefix` |`""` diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java index e3a9080e5d4..e0bf0b09098 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java @@ -82,10 +82,11 @@ private GCSEmitter(ExtensionConfig extensionConfig, GCSEmitterConfig config, Sto } private static Storage buildStorage(GCSEmitterConfig config) { - return StorageOptions.newBuilder() - .setProjectId(config.projectId()) - .build() - .getService(); + StorageOptions.Builder builder = StorageOptions.newBuilder(); + if (!StringUtils.isBlank(config.projectId())) { + builder.setProjectId(config.projectId()); + } + return builder.build().getService(); } @Override diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterConfig.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterConfig.java index a7408ba4b32..15a3264a557 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterConfig.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterConfig.java @@ -34,9 +34,6 @@ public static GCSEmitterConfig load(final String json) } public void validate() throws TikaConfigException { - if (projectId == null || projectId.isBlank()) { - throw new TikaConfigException("'projectId' must not be empty"); - } if (bucket == null || bucket.isBlank()) { throw new TikaConfigException("'bucket' must not be empty"); } diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java index 1f45615b411..82fc436d070 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java @@ -38,9 +38,10 @@ import org.apache.tika.pipes.fetcher.gcs.config.GCSFetcherConfig; import org.apache.tika.plugins.AbstractTikaExtension; import org.apache.tika.plugins.ExtensionConfig; +import org.apache.tika.utils.StringUtils; /** - * Fetches files from google cloud storage. Must set projectId and bucket via the config. + * Fetches files from google cloud storage. Must set bucket via the config; projectId is optional. */ public class GCSFetcher extends AbstractTikaExtension implements Fetcher { @@ -64,12 +65,12 @@ public static GCSFetcher build(ExtensionConfig extensionConfig) throws IOExcepti private void initialize() throws TikaConfigException { ConfigValidator.mustNotBeEmpty("bucket", config.getBucket()); - ConfigValidator.mustNotBeEmpty("projectId", config.getProjectId()); - storage = StorageOptions.newBuilder() - .setProjectId(config.getProjectId()) - .build() - .getService(); + StorageOptions.Builder builder = StorageOptions.newBuilder(); + if (!StringUtils.isBlank(config.getProjectId())) { + builder.setProjectId(config.getProjectId()); + } + storage = builder.build().getService(); } @Override diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/iterator/gcs/GCSPipesIterator.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/iterator/gcs/GCSPipesIterator.java index 8fb6df1547f..45804e3c896 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/iterator/gcs/GCSPipesIterator.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/main/java/org/apache/tika/pipes/iterator/gcs/GCSPipesIterator.java @@ -50,16 +50,12 @@ private GCSPipesIterator(GCSPipesIteratorConfig config, ExtensionConfig extensio if (StringUtils.isBlank(config.getBucket())) { throw new TikaConfigException("bucket must not be empty"); } - if (StringUtils.isBlank(config.getProjectId())) { - throw new TikaConfigException("projectId must not be empty"); - } - // Initialize the GCS client - this.storage = StorageOptions - .newBuilder() - .setProjectId(config.getProjectId()) - .build() - .getService(); + StorageOptions.Builder builder = StorageOptions.newBuilder(); + if (!StringUtils.isBlank(config.getProjectId())) { + builder.setProjectId(config.getProjectId()); + } + this.storage = builder.build().getService(); } public static GCSPipesIterator build(ExtensionConfig extensionConfig) throws IOException, TikaConfigException { diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/ConfigExamplesTest.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/ConfigExamplesTest.java index 52ce0a1fe56..eab655c387f 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/ConfigExamplesTest.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/ConfigExamplesTest.java @@ -17,6 +17,7 @@ package org.apache.tika.pipes.gcs; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import com.fasterxml.jackson.databind.JsonNode; import org.junit.jupiter.api.Test; @@ -41,7 +42,8 @@ public void testGCSFetcherConfig() throws Exception { JsonNode inner = innerComponent(readExample("gcs-fetcher.json"), "fetchers", "gcsf", "gcs-fetcher"); GCSFetcherConfig config = GCSFetcherConfig.load(inner.toString()); - assertEquals("my-gcp-project", config.getProjectId()); + // The fetcher example leaves projectId out to show it is optional. + assertNull(config.getProjectId()); assertEquals("my-tika-input", config.getBucket()); } diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java new file mode 100644 index 00000000000..cb7ba0c1401 --- /dev/null +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/java/org/apache/tika/pipes/gcs/GCSProjectIdTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.gcs; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.exception.TikaConfigException; +import org.apache.tika.pipes.emitter.gcs.GCSEmitter; +import org.apache.tika.pipes.fetcher.gcs.GCSFetcher; +import org.apache.tika.pipes.iterator.gcs.GCSPipesIterator; +import org.apache.tika.plugins.ExtensionConfig; + +/** + * projectId is optional: reading, writing and listing objects need no project, so each + * component builds without one. Building makes no request, so these need no credentials. + */ +public class GCSProjectIdTest { + + private static final String BUCKET_ONLY = "{\"bucket\": \"bucket\"}"; + private static final String WITH_PROJECT = "{\"bucket\": \"bucket\", \"projectId\": \"project\"}"; + + @Test + public void testFetcherWithoutProjectId() throws Exception { + assertNotNull(GCSFetcher.build(new ExtensionConfig("gcs", "gcs-fetcher", BUCKET_ONLY))); + } + + @Test + public void testFetcherWithProjectId() throws Exception { + assertNotNull(GCSFetcher.build(new ExtensionConfig("gcs", "gcs-fetcher", WITH_PROJECT))); + } + + @Test + public void testFetcherRequiresBucket() { + assertThrows(TikaConfigException.class, + () -> GCSFetcher.build(new ExtensionConfig("gcs", "gcs-fetcher", "{}"))); + } + + @Test + public void testEmitterWithoutProjectId() throws Exception { + assertNotNull(GCSEmitter.build(new ExtensionConfig("gcs", "gcs-emitter", BUCKET_ONLY))); + } + + @Test + public void testEmitterWithProjectId() throws Exception { + assertNotNull(GCSEmitter.build(new ExtensionConfig("gcs", "gcs-emitter", WITH_PROJECT))); + } + + @Test + public void testEmitterRequiresBucket() { + assertThrows(TikaConfigException.class, + () -> GCSEmitter.build(new ExtensionConfig("gcs", "gcs-emitter", "{}"))); + } + + @Test + public void testIteratorWithoutProjectId() throws Exception { + assertNotNull(GCSPipesIterator.build(new ExtensionConfig("gcs", "gcs-pipes-iterator", BUCKET_ONLY))); + } + + @Test + public void testIteratorWithProjectId() throws Exception { + assertNotNull(GCSPipesIterator.build(new ExtensionConfig("gcs", "gcs-pipes-iterator", WITH_PROJECT))); + } + + @Test + public void testIteratorRequiresBucket() { + assertThrows(TikaConfigException.class, + () -> GCSPipesIterator.build(new ExtensionConfig("gcs", "gcs-pipes-iterator", "{}"))); + } +} diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/resources/config-examples/gcs-fetcher.json b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/resources/config-examples/gcs-fetcher.json index 89ab85eed3b..177445ac285 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/resources/config-examples/gcs-fetcher.json +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/src/test/resources/config-examples/gcs-fetcher.json @@ -2,7 +2,6 @@ "fetchers": { "gcsf": { "gcs-fetcher": { - "projectId": "my-gcp-project", "bucket": "my-tika-input", "extractUserMetadata": true, "spoolToTemp": true