From 604de4b97136cb7a4a7391839897d9b3bb3351dd Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Sat, 29 Jan 2022 17:43:38 +0800 Subject: [PATCH] Allow to configure metadata store URL in functions_worker.yml (#13782) --- conf/functions_worker.yml | 11 +++++++++-- .../org/apache/pulsar/broker/PulsarService.java | 2 +- .../pulsar/functions/worker/WorkerConfig.java | 16 ++++++++++++++++ .../apache/pulsar/functions/worker/Worker.java | 3 ++- site2/docs/functions-runtime.md | 2 +- site2/docs/functions-worker.md | 4 ++-- site2/website-next/docs/functions-runtime.md | 2 +- site2/website-next/docs/functions-worker.md | 4 ++-- 8 files changed, 34 insertions(+), 10 deletions(-) diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index c68c615aaabf5..392ad4172e302 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -26,8 +26,12 @@ workerHostname: localhost workerPort: 6750 workerPortTls: 6751 -# Configuration Store connection string -configurationStoreServers: localhost:2181 +# The Configuration metadata store url +# Examples: +# * zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181 +# * my-zk-1:2181,my-zk-2:2181,my-zk-3:2181 (will default to ZooKeeper when the schema is not specified) +# * zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181/my-chroot-path (to add a ZK chroot path) +configurationMetadataStoreUrl: zk:localhost:2181 # ZooKeeper session timeout in milliseconds zooKeeperSessionTimeoutMillis: 30000 # ZooKeeper operation timeout in seconds @@ -328,3 +332,6 @@ validateConnectorConfig: false # Whether to initialize distributed log metadata by runtime. # If it is set to true, you must ensure that it has been initialized by "bin/pulsar initialize-cluster-metadata" command. initializedDlogMetadata: false + +### --- Deprecated settings --- ### +configurationStoreServers: localhost:2181 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 6956ccd04a827..bb793722d1eaf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1608,7 +1608,7 @@ public static WorkerConfig initializeWorkerConfigFromBrokerConfig(ServiceConfigu workerConfig.setAuthenticationProviders(brokerConfig.getAuthenticationProviders()); workerConfig.setAuthorizationEnabled(brokerConfig.isAuthorizationEnabled()); workerConfig.setAuthorizationProvider(brokerConfig.getAuthorizationProvider()); - workerConfig.setConfigurationStoreServers(brokerConfig.getConfigurationMetadataStoreUrl()); + workerConfig.setConfigurationMetadataStoreUrl(brokerConfig.getConfigurationMetadataStoreUrl()); workerConfig.setZooKeeperSessionTimeoutMillis(brokerConfig.getZooKeeperSessionTimeoutMillis()); workerConfig.setZooKeeperOperationTimeoutSeconds(brokerConfig.getZooKeeperOperationTimeoutSeconds()); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java index abb5b5e913b1a..c65e6ce31e48a 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java @@ -144,9 +144,17 @@ public class WorkerConfig implements Serializable, PulsarConfiguration { @FieldContext( category = CATEGORY_WORKER, required = false, + deprecated = true, doc = "Configuration store connection string (as a comma-separated list)" ) + @Deprecated private String configurationStoreServers; + @FieldContext( + category = CATEGORY_WORKER, + required = false, + doc = "Configuration store connection string (as a comma-separated list)" + ) + private String configurationMetadataStoreUrl; @FieldContext( category = CATEGORY_WORKER, doc = "ZooKeeper session timeout in milliseconds" @@ -633,6 +641,14 @@ public static String unsafeLocalhostResolve() { } } + public String getConfigurationMetadataStoreUrl() { + if (StringUtils.isNotBlank(configurationMetadataStoreUrl)) { + return configurationMetadataStoreUrl; + } else { + return configurationStoreServers; + } + } + @Override public void setProperties(Properties properties) { this.properties = properties; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Worker.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Worker.java index 929e62e862621..a277e3b6083dc 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Worker.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Worker.java @@ -73,7 +73,8 @@ private AuthorizationService getAuthorizationService() throws PulsarServerExcept log.info("starting configuration cache service"); try { - configMetadataStore = PulsarResources.createMetadataStore(workerConfig.getConfigurationStoreServers(), + configMetadataStore = PulsarResources.createMetadataStore( + workerConfig.getConfigurationMetadataStoreUrl(), (int) workerConfig.getZooKeeperSessionTimeoutMillis()); } catch (IOException e) { throw new PulsarServerException(e); diff --git a/site2/docs/functions-runtime.md b/site2/docs/functions-runtime.md index bdbd658dd9814..f65393a27aab1 100644 --- a/site2/docs/functions-runtime.md +++ b/site2/docs/functions-runtime.md @@ -270,7 +270,7 @@ For example, if you use token authentication, you need to configure the followin ```Yaml clientAuthenticationPlugin: org.apache.pulsar.client.impl.auth.AuthenticationToken clientAuthenticationParameters: file:///etc/pulsar/token/admin-token.txt -configurationStoreServers: zookeeper-cluster:2181 # auth requires a connection to zookeeper +configurationMetadataStoreUrl: zk:zookeeper-cluster:2181 # auth requires a connection to zookeeper authenticationProviders: - "org.apache.pulsar.broker.authentication.AuthenticationProviderToken" authorizationEnabled: true diff --git a/site2/docs/functions-worker.md b/site2/docs/functions-worker.md index 36cf86419d5a9..c58d5a436d1a9 100644 --- a/site2/docs/functions-worker.md +++ b/site2/docs/functions-worker.md @@ -216,12 +216,12 @@ properties: ##### Enable Authorization Provider -To enable authorization on Functions Worker, you need to configure `authorizationEnabled`, `authorizationProvider` and `configurationStoreServers`. The authentication provider connects to `configurationStoreServers` to receive namespace policies. +To enable authorization on Functions Worker, you need to configure `authorizationEnabled`, `authorizationProvider` and `configurationMetadataStoreUrl`. The authentication provider connects to `configurationMetadataStoreUrl` to receive namespace policies. ```yaml authorizationEnabled: true authorizationProvider: org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider -configurationStoreServers: +configurationMetadataStoreUrl: : ``` You should also configure a list of superuser roles. The superuser roles are able to access any admin API. The following is a configuration example. diff --git a/site2/website-next/docs/functions-runtime.md b/site2/website-next/docs/functions-runtime.md index 96364364e6eaf..eff05e4e7e675 100644 --- a/site2/website-next/docs/functions-runtime.md +++ b/site2/website-next/docs/functions-runtime.md @@ -292,7 +292,7 @@ For example, if you use token authentication, you need to configure the followin clientAuthenticationPlugin: org.apache.pulsar.client.impl.auth.AuthenticationToken clientAuthenticationParameters: file:///etc/pulsar/token/admin-token.txt -configurationStoreServers: zookeeper-cluster:2181 # auth requires a connection to zookeeper +configurationMetadataStoreUrl: zk:zookeeper-cluster:2181 # auth requires a connection to zookeeper authenticationProviders: - "org.apache.pulsar.broker.authentication.AuthenticationProviderToken" authorizationEnabled: true diff --git a/site2/website-next/docs/functions-worker.md b/site2/website-next/docs/functions-worker.md index 35867f7557492..32c9abb0d5b71 100644 --- a/site2/website-next/docs/functions-worker.md +++ b/site2/website-next/docs/functions-worker.md @@ -235,13 +235,13 @@ properties: ##### Enable Authorization Provider -To enable authorization on Functions Worker, you need to configure `authorizationEnabled`, `authorizationProvider` and `configurationStoreServers`. The authentication provider connects to `configurationStoreServers` to receive namespace policies. +To enable authorization on Functions Worker, you need to configure `authorizationEnabled`, `authorizationProvider` and `configurationMetadataStoreUrl`. The authentication provider connects to `configurationMetadataStoreUrl` to receive namespace policies. ```yaml authorizationEnabled: true authorizationProvider: org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider -configurationStoreServers: +configurationMetadataStoreUrl: : ```