2023-02-09T16:53:12+00:00 startup service overlord Setting druid.host=10.233.83.196 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.node.type=overlord in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.runner.javaOptsArray=["-server", "-Xms8g", "-Xmx8g", "-XX:MaxDirectMemorySize=4g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-XX:+ExitOnOutOfMemoryError", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.logs.type=s3 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.processing.intermediaryData.storage.type=deepstore in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.fork.property.druid.processing.numMergeBuffers=2 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.extensions.loadList=["postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-kubernetes-extensions", "druid-kubernetes-overlord-extensions", "druid-s3-extensions"] in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.storage.baseKey=sjc4 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.storage.type=metadata in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.queue.maxSize=30 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.storage.archiveBaseKey=sjc4 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.logs.s3Bucket=zpa-dev-druid in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.metadata.storage.type=postgresql in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.emitter.http.recipientBaseUrl=http://druid_exporter_url:druid_exporter_port/druid in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.fork.property.druid.processing.numThreads=1 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.storage.bucket=zpa-dev-druid in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.discovery.type=k8s in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.worker.capacity=4 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.task.encapsulatedTask=true in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.metadata.storage.connector.connectURI=jdbc:postgresql://druid1-pg-rw:5432/druid1 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.server.http.numThreads=60 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.discovery.k8s.clusterIdentifier=druid1 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.storage.sse.type=s3 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.storage.archiveBucket=zpa-dev-druid in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.serverview.type=http in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.runner.namespace=druid1 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.runner.disableClientProxy=true in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.storage.type=s3 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.zk.service.enabled=false in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.metadata.storage.connector.user=druid in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.host=10.233.83.196 in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.logs.directory=/opt/data/indexing-logs in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.task.baseTaskDir=var/druid/task in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.emitter=noop in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.logs.s3Prefix=sjc4/indexing-logs in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.metadata.storage.connector.password=REDACTED in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.emitter.logging.logLevel=debug in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties Setting druid.indexer.runner.type=k8s in /tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties WARNING: An illegal reflective access operation has occurred WARNING: Illegal reflective access by com.google.inject.internal.cglib.core.$ReflectUtils$1 (file:/opt/druid/lib/guice-4.1.0.jar) to method java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int,java.security.ProtectionDomain) WARNING: Please consider reporting this to the maintainers of com.google.inject.internal.cglib.core.$ReflectUtils$1 WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations WARNING: All illegal access operations will be denied in a future release 2023-02-09T16:53:14,761 INFO [main] org.hibernate.validator.internal.util.Version - HV000001: Hibernate Validator 5.2.5.Final 2023-02-09T16:53:15,471 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [postgresql-metadata-storage], jars: checker-qual-3.5.0.jar, postgresql-42.4.1.jar, postgresql-metadata-storage-25.0.0.jar 2023-02-09T16:53:15,474 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [druid-kafka-indexing-service], jars: druid-kafka-indexing-service-25.0.0.jar, kafka-clients-3.3.1.jar, lz4-java-1.8.0.jar, snappy-java-1.1.8.4.jar, zstd-jni-1.5.2-3.jar 2023-02-09T16:53:15,475 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [druid-datasketches], jars: commons-math3-3.6.1.jar, druid-datasketches-25.0.0.jar 2023-02-09T16:53:15,478 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [druid-kubernetes-extensions], jars: bcpkix-jdk15on-1.66.jar, bcprov-ext-jdk15on-1.68.jar, bcprov-jdk15on-1.68.jar, bucket4j-core-4.10.0.jar, caffeine-2.8.0.jar, checker-qual-2.10.0.jar, client-java-11.0.1.jar, client-java-api-11.0.1.jar, client-java-extended-11.0.1.jar, client-java-proto-11.0.1.jar, commons-codec-1.13.jar, commons-collections4-4.2.jar, commons-io-2.11.0.jar, commons-lang3-3.12.0.jar, druid-kubernetes-extensions-25.0.0.jar, error_prone_annotations-2.11.0.jar, gson-2.8.6.jar, gson-fire-1.8.5.jar, javax.annotation-api-1.3.2.jar, joda-convert-2.2.1.jar, joda-time-2.10.5.jar, jose4j-0.7.3.jar, jsr305-2.0.1.jar, logging-interceptor-3.14.9.jar, okhttp-3.14.9.jar, okio-1.17.2.jar, protobuf-java-3.21.7.jar, simpleclient-0.9.0.jar, simpleclient_common-0.9.0.jar, simpleclient_httpserver-0.9.0.jar, slf4j-api-1.7.36.jar, snakeyaml-1.27.jar, swagger-annotations-1.6.2.jar, zjsonpatch-0.4.11.jar 2023-02-09T16:53:15,495 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [druid-kubernetes-overlord-extensions], jars: automaton-1.11-8.jar, druid-kubernetes-overlord-extensions-25.0.0.jar, generex-1.0.2.jar, jackson-dataformat-yaml-2.10.5.jar, jackson-datatype-jsr310-2.10.5.jar, kubernetes-client-5.12.2.jar, kubernetes-model-admissionregistration-5.12.2.jar, kubernetes-model-apiextensions-5.12.2.jar, kubernetes-model-apps-5.12.2.jar, kubernetes-model-autoscaling-5.12.2.jar, kubernetes-model-batch-5.12.2.jar, kubernetes-model-certificates-5.12.2.jar, kubernetes-model-common-5.12.2.jar, kubernetes-model-coordination-5.12.2.jar, kubernetes-model-core-5.12.2.jar, kubernetes-model-discovery-5.12.2.jar, kubernetes-model-events-5.12.2.jar, kubernetes-model-extensions-5.12.2.jar, kubernetes-model-flowcontrol-5.12.2.jar, kubernetes-model-metrics-5.12.2.jar, kubernetes-model-networking-5.12.2.jar, kubernetes-model-node-5.12.2.jar, kubernetes-model-policy-5.12.2.jar, kubernetes-model-rbac-5.12.2.jar, kubernetes-model-scheduling-5.12.2.jar, kubernetes-model-storageclass-5.12.2.jar, logging-interceptor-3.12.12.jar, okhttp-3.12.12.jar, okio-1.15.0.jar, slf4j-api-1.7.36.jar, snakeyaml-1.26.jar, zjsonpatch-0.3.0.jar 2023-02-09T16:53:15,556 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [druid-s3-extensions], jars: aws-java-sdk-core-1.12.317.jar, aws-java-sdk-sts-1.12.317.jar, commons-codec-1.13.jar, commons-logging-1.1.1.jar, druid-s3-extensions-25.0.0.jar, httpclient-4.5.13.jar, httpcore-4.4.11.jar, ion-java-1.0.2.jar, jackson-dataformat-cbor-2.10.5.jar, jmespath-java-1.12.317.jar, joda-time-2.10.5.jar 2023-02-09T16:53:15,954 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [postgresql-metadata-storage], jars: checker-qual-3.5.0.jar, postgresql-42.4.1.jar, postgresql-metadata-storage-25.0.0.jar 2023-02-09T16:53:15,955 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [druid-kafka-indexing-service], jars: druid-kafka-indexing-service-25.0.0.jar, kafka-clients-3.3.1.jar, lz4-java-1.8.0.jar, snappy-java-1.1.8.4.jar, zstd-jni-1.5.2-3.jar 2023-02-09T16:53:15,957 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [druid-datasketches], jars: commons-math3-3.6.1.jar, druid-datasketches-25.0.0.jar 2023-02-09T16:53:15,962 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [druid-kubernetes-extensions], jars: bcpkix-jdk15on-1.66.jar, bcprov-ext-jdk15on-1.68.jar, bcprov-jdk15on-1.68.jar, bucket4j-core-4.10.0.jar, caffeine-2.8.0.jar, checker-qual-2.10.0.jar, client-java-11.0.1.jar, client-java-api-11.0.1.jar, client-java-extended-11.0.1.jar, client-java-proto-11.0.1.jar, commons-codec-1.13.jar, commons-collections4-4.2.jar, commons-io-2.11.0.jar, commons-lang3-3.12.0.jar, druid-kubernetes-extensions-25.0.0.jar, error_prone_annotations-2.11.0.jar, gson-2.8.6.jar, gson-fire-1.8.5.jar, javax.annotation-api-1.3.2.jar, joda-convert-2.2.1.jar, joda-time-2.10.5.jar, jose4j-0.7.3.jar, jsr305-2.0.1.jar, logging-interceptor-3.14.9.jar, okhttp-3.14.9.jar, okio-1.17.2.jar, protobuf-java-3.21.7.jar, simpleclient-0.9.0.jar, simpleclient_common-0.9.0.jar, simpleclient_httpserver-0.9.0.jar, slf4j-api-1.7.36.jar, snakeyaml-1.27.jar, swagger-annotations-1.6.2.jar, zjsonpatch-0.4.11.jar 2023-02-09T16:53:15,963 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [druid-kubernetes-overlord-extensions], jars: automaton-1.11-8.jar, druid-kubernetes-overlord-extensions-25.0.0.jar, generex-1.0.2.jar, jackson-dataformat-yaml-2.10.5.jar, jackson-datatype-jsr310-2.10.5.jar, kubernetes-client-5.12.2.jar, kubernetes-model-admissionregistration-5.12.2.jar, kubernetes-model-apiextensions-5.12.2.jar, kubernetes-model-apps-5.12.2.jar, kubernetes-model-autoscaling-5.12.2.jar, kubernetes-model-batch-5.12.2.jar, kubernetes-model-certificates-5.12.2.jar, kubernetes-model-common-5.12.2.jar, kubernetes-model-coordination-5.12.2.jar, kubernetes-model-core-5.12.2.jar, kubernetes-model-discovery-5.12.2.jar, kubernetes-model-events-5.12.2.jar, kubernetes-model-extensions-5.12.2.jar, kubernetes-model-flowcontrol-5.12.2.jar, kubernetes-model-metrics-5.12.2.jar, kubernetes-model-networking-5.12.2.jar, kubernetes-model-node-5.12.2.jar, kubernetes-model-policy-5.12.2.jar, kubernetes-model-rbac-5.12.2.jar, kubernetes-model-scheduling-5.12.2.jar, kubernetes-model-storageclass-5.12.2.jar, logging-interceptor-3.12.12.jar, okhttp-3.12.12.jar, okio-1.15.0.jar, slf4j-api-1.7.36.jar, snakeyaml-1.26.jar, zjsonpatch-0.3.0.jar 2023-02-09T16:53:15,965 INFO [main] org.apache.druid.guice.ExtensionsLoader - Loading extension [druid-s3-extensions], jars: aws-java-sdk-core-1.12.317.jar, aws-java-sdk-sts-1.12.317.jar, commons-codec-1.13.jar, commons-logging-1.1.1.jar, druid-s3-extensions-25.0.0.jar, httpclient-4.5.13.jar, httpcore-4.4.11.jar, ion-java-1.0.2.jar, jackson-dataformat-cbor-2.10.5.jar, jmespath-java-1.12.317.jar, joda-time-2.10.5.jar 2023-02-09T16:53:16,674 INFO [main] org.apache.druid.cli.CliPeon - Running peon in k8s mode 2023-02-09T16:53:17,578 INFO [main] org.apache.druid.guice.JsonConfigurator - Skipping druid.emitter.http.recipientBaseUrl property: one of it's prefixes is also used as a property key. Prefix: druid 2023-02-09T16:53:17,578 INFO [main] org.apache.druid.guice.JsonConfigurator - Skipping druid.emitter.logging.logLevel property: one of it's prefixes is also used as a property key. Prefix: druid 2023-02-09T16:53:17,596 WARN [main] org.apache.druid.k8s.discovery.K8sDiscoveryConfig - IF coordinator pods run in multiple namespaces, then you MUST provide coordinatorLeaderElectionConfigMapNamespace 2023-02-09T16:53:17,596 WARN [main] org.apache.druid.k8s.discovery.K8sDiscoveryConfig - IF overlord pods run in multiple namespaces, then you MUST provide overlordLeaderElectionConfigMapNamespace 2023-02-09T16:53:17,761 INFO [main] org.apache.druid.server.emitter.EmitterModule - Using emitter [NoopEmitter{}] for metrics and alerts, with dimensions [{version=25.0.0}]. 2023-02-09T16:53:17,778 WARN [main] org.apache.druid.java.util.metrics.AllocationMetricCollectors - Cannot initialize org.apache.druid.java.util.metrics.AllocationMetricCollector java.lang.reflect.InaccessibleObjectException: Unable to make public long[] com.sun.management.internal.HotSpotThreadImpl.getThreadAllocatedBytes(long[]) accessible: module jdk.management does not "exports com.sun.management.internal" to unnamed module @3383649e at java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:340) ~[?:?] at java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:280) ~[?:?] at java.lang.reflect.Method.checkCanSetAccessible(Method.java:198) ~[?:?] at java.lang.reflect.Method.setAccessible(Method.java:192) ~[?:?] at org.apache.druid.java.util.metrics.AllocationMetricCollectors.(AllocationMetricCollectors.java:41) ~[druid-core-25.0.0.jar:25.0.0] at org.apache.druid.java.util.metrics.JvmMonitor.(JvmMonitor.java:67) ~[druid-core-25.0.0.jar:25.0.0] at org.apache.druid.java.util.metrics.JvmMonitor.(JvmMonitor.java:59) ~[druid-core-25.0.0.jar:25.0.0] at org.apache.druid.server.metrics.MetricsModule.getJvmMonitor(MetricsModule.java:154) ~[druid-server-25.0.0.jar:25.0.0] at org.apache.druid.server.metrics.MetricsModule$$FastClassByGuice$$99ddce1b.invoke() ~[druid-server-25.0.0.jar:25.0.0] at com.google.inject.internal.ProviderMethod$FastClassProviderMethod.doProvision(ProviderMethod.java:264) ~[guice-4.1.0.jar:?] at com.google.inject.internal.ProviderMethod$Factory.provision(ProviderMethod.java:401) ~[guice-4.1.0.jar:?] at com.google.inject.internal.ProviderMethod$Factory.get(ProviderMethod.java:376) ~[guice-4.1.0.jar:?] at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1092) ~[guice-4.1.0.jar:?] at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40) ~[guice-4.1.0.jar:?] at org.apache.druid.guice.LifecycleScope$1.get(LifecycleScope.java:68) ~[druid-core-25.0.0.jar:25.0.0] at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:41) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InjectorImpl$2$1.call(InjectorImpl.java:1019) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1092) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InjectorImpl$2.get(InjectorImpl.java:1015) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InjectorImpl.getInstance(InjectorImpl.java:1054) ~[guice-4.1.0.jar:?] at org.apache.druid.server.metrics.MetricsModule.getMonitorScheduler(MetricsModule.java:113) ~[druid-server-25.0.0.jar:25.0.0] at org.apache.druid.server.metrics.MetricsModule$$FastClassByGuice$$99ddce1b.invoke() ~[druid-server-25.0.0.jar:25.0.0] at com.google.inject.internal.ProviderMethod$FastClassProviderMethod.doProvision(ProviderMethod.java:264) ~[guice-4.1.0.jar:?] at com.google.inject.internal.ProviderMethod$Factory.provision(ProviderMethod.java:401) ~[guice-4.1.0.jar:?] at com.google.inject.internal.ProviderMethod$Factory.get(ProviderMethod.java:376) ~[guice-4.1.0.jar:?] at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1092) ~[guice-4.1.0.jar:?] at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40) ~[guice-4.1.0.jar:?] at org.apache.druid.guice.LifecycleScope$1.get(LifecycleScope.java:68) ~[druid-core-25.0.0.jar:25.0.0] at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:41) ~[guice-4.1.0.jar:?] at com.google.inject.internal.FactoryProxy.get(FactoryProxy.java:56) ~[guice-4.1.0.jar:?] at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1092) ~[guice-4.1.0.jar:?] at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40) ~[guice-4.1.0.jar:?] at com.google.inject.internal.SingletonScope$1.get(SingletonScope.java:194) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:41) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InternalInjectorCreator$1.call(InternalInjectorCreator.java:205) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InternalInjectorCreator$1.call(InternalInjectorCreator.java:199) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1085) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InternalInjectorCreator.loadEagerSingletons(InternalInjectorCreator.java:199) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InternalInjectorCreator.injectDynamically(InternalInjectorCreator.java:180) ~[guice-4.1.0.jar:?] at com.google.inject.internal.InternalInjectorCreator.build(InternalInjectorCreator.java:110) ~[guice-4.1.0.jar:?] at com.google.inject.Guice.createInjector(Guice.java:99) ~[guice-4.1.0.jar:?] at com.google.inject.Guice.createInjector(Guice.java:73) ~[guice-4.1.0.jar:?] at com.google.inject.Guice.createInjector(Guice.java:62) ~[guice-4.1.0.jar:?] at org.apache.druid.initialization.ExtensionInjectorBuilder.build(ExtensionInjectorBuilder.java:49) ~[druid-server-25.0.0.jar:25.0.0] at org.apache.druid.initialization.ServerInjectorBuilder.build(ServerInjectorBuilder.java:118) ~[druid-server-25.0.0.jar:25.0.0] at org.apache.druid.initialization.ServerInjectorBuilder.makeServerInjector(ServerInjectorBuilder.java:73) ~[druid-server-25.0.0.jar:25.0.0] at org.apache.druid.cli.GuiceRunnable.makeInjector(GuiceRunnable.java:85) ~[druid-services-25.0.0.jar:25.0.0] at org.apache.druid.cli.CliPeon.run(CliPeon.java:308) ~[druid-services-25.0.0.jar:25.0.0] at org.apache.druid.cli.Main.main(Main.java:112) ~[druid-services-25.0.0.jar:25.0.0] 2023-02-09T16:53:17,783 INFO [main] org.apache.druid.server.metrics.MetricsModule - Loaded 3 monitors: org.apache.druid.java.util.metrics.JvmMonitor, org.apache.druid.query.ExecutorServiceMonitor, org.apache.druid.server.initialization.jetty.JettyServerModule$JettyMonitor 2023-02-09T16:53:17,791 INFO [main] org.apache.druid.cli.CliPeon - Starting up with processors [1], memory [8,303,607,808], maxMemory [8,303,607,808]. Properties follow. 2023-02-09T16:53:17,792 INFO [main] org.apache.druid.cli.CliPeon - * awt.toolkit: sun.awt.X11.XToolkit 2023-02-09T16:53:17,792 INFO [main] org.apache.druid.cli.CliPeon - * druid.coordinator.asOverlord.enabled: true 2023-02-09T16:53:17,792 INFO [main] org.apache.druid.cli.CliPeon - * druid.coordinator.asOverlord.overlordService: druid/overlord 2023-02-09T16:53:17,792 INFO [main] org.apache.druid.cli.CliPeon - * druid.coordinator.period: PT5S 2023-02-09T16:53:17,792 INFO [main] org.apache.druid.cli.CliPeon - * druid.coordinator.startDelay: PT10S 2023-02-09T16:53:17,792 INFO [main] org.apache.druid.cli.CliPeon - * druid.discovery.k8s.clusterIdentifier: druid1 2023-02-09T16:53:17,792 INFO [main] org.apache.druid.cli.CliPeon - * druid.discovery.type: k8s 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.emitter: noop 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.emitter.http.recipientBaseUrl: http://druid_exporter_url:druid_exporter_port/druid 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.emitter.logging.logLevel: debug 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.expressions.useStrictBooleans: true 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.extensions.loadList: ["postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-kubernetes-extensions", "druid-kubernetes-overlord-extensions", "druid-s3-extensions"] 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.global.http.eagerInitialization: false 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.host: 10.233.83.196 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.fork.property.druid.processing.buffer.sizeBytes: 100MiB 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.fork.property.druid.processing.numMergeBuffers: 2 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.fork.property.druid.processing.numThreads: 1 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.logs.directory: /opt/data/indexing-logs 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.logs.s3Bucket: zpa-dev-druid 2023-02-09T16:53:17,793 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.logs.s3Prefix: sjc4/indexing-logs 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.logs.type: s3 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.queue.maxSize: 30 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.queue.startDelay: PT5S 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.runner.disableClientProxy: true 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.runner.javaOptsArray: ["-server", "-Xms8g", "-Xmx8g", "-XX:MaxDirectMemorySize=4g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-XX:+ExitOnOutOfMemoryError", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.runner.namespace: druid1 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.runner.type: k8s 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.storage.type: metadata 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.task.baseTaskDir: var/druid/task 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexer.task.encapsulatedTask: true 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.indexing.doubleStorage: double 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.lookup.enableLookupSyncOnStartup: false 2023-02-09T16:53:17,794 INFO [main] org.apache.druid.cli.CliPeon - * druid.manager.segments.pollDuration: PT5S 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.metadata.storage.connector.connectURI: jdbc:postgresql://druid1-pg-rw:5432/druid1 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.metadata.storage.connector.host: localhost 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.metadata.storage.connector.password: 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.metadata.storage.connector.port: 1527 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.metadata.storage.connector.user: druid 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.metadata.storage.type: postgresql 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.monitoring.monitors: ["org.apache.druid.java.util.metrics.JvmMonitor"] 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.node.type: overlord 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.plaintextPort: 8100 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.port: 8100 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.processing.intermediaryData.storage.type: deepstore 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.selectors.coordinator.serviceName: druid/coordinator 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.selectors.indexing.serviceName: druid/overlord 2023-02-09T16:53:17,795 INFO [main] org.apache.druid.cli.CliPeon - * druid.server.hiddenProperties: ["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.server.http.numThreads: 60 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.serverview.type: http 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.service: druid/coordinator 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.sql.enable: true 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.sql.planner.useGroupingSetForExactDistinct: true 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.startup.logging.logProperties: true 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.storage.archiveBaseKey: sjc4 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.storage.archiveBucket: zpa-dev-druid 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.storage.baseKey: sjc4 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.storage.bucket: zpa-dev-druid 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.storage.sse.type: s3 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.storage.storageDirectory: var/druid/segments 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.storage.type: s3 2023-02-09T16:53:17,796 INFO [main] org.apache.druid.cli.CliPeon - * druid.task.executor.enableTlsPort: false 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * druid.task.executor.tlsPort: -1 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * druid.tlsPort: -1 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * druid.worker.capacity: 4 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * druid.zk.paths.base: /druid 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * druid.zk.service.enabled: false 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * druid.zk.service.host: localhost 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * file.encoding: UTF-8 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * file.separator: / 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * java.awt.graphicsenv: sun.awt.X11GraphicsEnvironment 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * java.awt.printerjob: sun.print.PSPrinterJob 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * java.class.path: /tmp/conf/druid/cluster/_common:/tmp/conf/druid/cluster/master/coordinator-overlord:lib/FastInfoset-1.2.15.jar:lib/RoaringBitmap-0.9.0.jar:lib/accessors-smart-1.2.jar:lib/aether-api-0.9.0.M2.jar:lib/aether-connector-file-0.9.0.M2.jar:lib/aether-connector-okhttp-0.0.9.jar:lib/aether-impl-0.9.0.M2.jar:lib/aether-spi-0.9.0.M2.jar:lib/aether-util-0.9.0.M2.jar:lib/aggdesigner-algorithm-6.0.jar:lib/airline-2.8.4.jar:lib/airline-io-2.8.4.jar:lib/antlr4-runtime-4.5.1.jar:lib/aopalliance-1.0.jar:lib/asm-9.3.jar:lib/asm-analysis-9.3.jar:lib/asm-commons-9.3.jar:lib/asm-tree-9.3.jar:lib/async-http-client-2.5.3.jar:lib/async-http-client-netty-utils-2.5.3.jar:lib/audience-annotations-0.5.0.jar:lib/avatica-core-1.17.0.jar:lib/avatica-metrics-1.17.0.jar:lib/avatica-server-1.17.0.jar:lib/aws-java-sdk-core-1.12.317.jar:lib/aws-java-sdk-ec2-1.12.317.jar:lib/aws-java-sdk-kms-1.12.317.jar:lib/aws-java-sdk-s3-1.12.317.jar:lib/aws-java-sdk-sts-1.12.317.jar:lib/caffeine-2.8.0.jar:lib/calcite-core-1.21.0.jar:lib/calcite-linq4j-1.21.0.jar:lib/checker-qual-2.5.7.jar:lib/classmate-1.1.0.jar:lib/commons-beanutils-1.9.4.jar:lib/commons-codec-1.13.jar:lib/commons-collections-3.2.2.jar:lib/commons-collections4-4.2.jar:lib/commons-compiler-3.0.11.jar:lib/commons-compress-1.21.jar:lib/commons-dbcp2-2.0.1.jar:lib/commons-io-2.11.0.jar:lib/commons-lang-2.6.jar:lib/commons-lang3-3.12.0.jar:lib/commons-logging-1.1.1.jar:lib/commons-math3-3.6.1.jar:lib/commons-pool2-2.2.jar:lib/commons-text-1.10.0.jar:lib/compress-lzf-1.0.4.jar:lib/config-magic-0.9.jar:lib/cron-scheduler-0.1.jar:lib/curator-client-5.4.0.jar:lib/curator-framework-5.4.0.jar:lib/curator-recipes-5.4.0.jar:lib/curator-x-discovery-5.4.0.jar:lib/datasketches-java-3.2.0.jar:lib/datasketches-memory-2.0.0.jar:lib/derby-10.14.2.0.jar:lib/derbyclient-10.14.2.0.jar:lib/derbynet-10.14.2.0.jar:lib/disruptor-3.3.6.jar:lib/druid-aws-common-25.0.0.jar:lib/druid-core-25.0.0.jar:lib/druid-gcp-common-25.0.0.jar:lib/druid-hll-25.0.0.jar:lib/druid-indexing-hadoop-25.0.0.jar:lib/druid-indexing-service-25.0.0.jar:lib/druid-processing-25.0.0.jar:lib/druid-server-25.0.0.jar:lib/druid-services-25.0.0.jar:lib/druid-sql-25.0.0.jar:lib/error_prone_annotations-2.11.0.jar:lib/esri-geometry-api-2.2.0.jar:lib/extendedset-25.0.0.jar:lib/fastutil-8.5.4.jar:lib/fastutil-core-8.5.4.jar:lib/fastutil-extra-8.5.4.jar:lib/google-api-client-1.26.0.jar:lib/google-http-client-1.26.0.jar:lib/google-http-client-jackson2-1.26.0.jar:lib/google-oauth-client-1.26.0.jar:lib/guava-16.0.1.jar:lib/guice-4.1.0.jar:lib/guice-assistedinject-4.1.0.jar:lib/guice-multibindings-4.1.0.jar:lib/guice-servlet-4.1.0.jar:lib/hibernate-validator-5.2.5.Final.jar:lib/httpclient-4.5.13.jar:lib/httpcore-4.4.11.jar:lib/icu4j-55.1.jar:lib/ion-java-1.0.2.jar:lib/ipaddress-5.3.4.jar:lib/istack-commons-runtime-3.0.7.jar:lib/j2objc-annotations-1.1.jar:lib/jackson-annotations-2.10.5.jar:lib/jackson-core-2.10.5.jar:lib/jackson-databind-2.10.5.1.jar:lib/jackson-dataformat-cbor-2.10.5.jar:lib/jackson-dataformat-smile-2.10.5.jar:lib/jackson-datatype-guava-2.10.5.jar:lib/jackson-datatype-joda-2.10.5.jar:lib/jackson-jaxrs-base-2.10.5.jar:lib/jackson-jaxrs-json-provider-2.10.5.jar:lib/jackson-jaxrs-smile-provider-2.10.5.jar:lib/jackson-jq-0.0.10.jar:lib/jackson-module-guice-2.10.5.jar:lib/jackson-module-jaxb-annotations-2.10.5.jar:lib/jakarta.activation-api-1.2.1.jar:lib/jakarta.inject-api-1.0.3.jar:lib/jakarta.xml.bind-api-2.3.2.jar:lib/janino-3.0.11.jar:lib/javax.activation-1.2.0.jar:lib/javax.activation-api-1.2.0.jar:lib/javax.el-3.0.0.jar:lib/javax.el-api-3.0.0.jar:lib/javax.inject-1.jar:lib/javax.servlet-api-3.1.0.jar:lib/jaxb-api-2.3.1.jar:lib/jaxb-runtime-2.3.1.jar:lib/jboss-logging-3.2.1.Final.jar:lib/jcl-over-slf4j-1.7.36.jar:lib/jcodings-1.0.43.jar:lib/jdbi-2.63.1.jar:lib/jersey-core-1.19.4.jar:lib/jersey-guice-1.19.4.jar:lib/jersey-server-1.19.4.jar:lib/jersey-servlet-1.19.4.jar:lib/jetty-client-9.4.48.v20220622.jar:lib/jetty-continuation-9.4.48.v20220622.jar:lib/jetty-http-9.4.48.v20220622.jar:lib/jetty-io-9.4.48.v20220622.jar:lib/jetty-proxy-9.4.48.v20220622.jar:lib/jetty-rewrite-9.4.48.v20220622.jar:lib/jetty-security-9.4.48.v20220622.jar:lib/jetty-server-9.4.48.v20220622.jar:lib/jetty-servlet-9.4.48.v20220622.jar:lib/jetty-servlets-9.4.48.v20220622.jar:lib/jetty-util-9.4.48.v20220622.jar:lib/jetty-util-ajax-9.4.48.v20220622.jar:lib/jmespath-java-1.12.317.jar:lib/jna-4.5.1.jar:lib/joda-time-2.10.5.jar:lib/joni-2.1.27.jar:lib/json-path-2.3.0.jar:lib/json-smart-2.3.jar:lib/jsr305-2.0.1.jar:lib/jsr311-api-1.1.1.jar:lib/log4j-1.2-api-2.18.0.jar:lib/log4j-api-2.18.0.jar:lib/log4j-core-2.18.0.jar:lib/log4j-jul-2.18.0.jar:lib/log4j-slf4j-impl-2.18.0.jar:lib/lz4-java-1.8.0.jar:lib/maven-aether-provider-3.1.1.jar:lib/maven-artifact-3.6.0.jar:lib/maven-model-3.1.1.jar:lib/maven-model-builder-3.1.1.jar:lib/maven-repository-metadata-3.1.1.jar:lib/maven-settings-3.1.1.jar:lib/maven-settings-builder-3.1.1.jar:lib/metrics-core-4.0.0.jar:lib/netty-3.10.6.Final.jar:lib/netty-buffer-4.1.68.Final.jar:lib/netty-codec-4.1.68.Final.jar:lib/netty-codec-dns-4.1.68.Final.jar:lib/netty-codec-http-4.1.68.Final.jar:lib/netty-codec-socks-4.1.68.Final.jar:lib/netty-common-4.1.68.Final.jar:lib/netty-handler-4.1.68.Final.jar:lib/netty-handler-proxy-4.1.68.Final.jar:lib/netty-reactive-streams-2.0.0.jar:lib/netty-resolver-4.1.68.Final.jar:lib/netty-resolver-dns-4.1.68.Final.jar:lib/netty-transport-4.1.68.Final.jar:lib/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar:lib/netty-transport-native-unix-common-4.1.68.Final.jar:lib/okhttp-1.0.2.jar:lib/opencsv-4.6.jar:lib/plexus-interpolation-1.19.jar:lib/plexus-utils-3.0.24.jar:lib/protobuf-java-3.21.7.jar:lib/reactive-streams-1.0.2.jar:lib/resilience4j-bulkhead-1.3.1.jar:lib/resilience4j-core-1.3.1.jar:lib/rhino-1.7.11.jar:lib/shims-0.9.0.jar:lib/sigar-1.6.5.132.jar:lib/slf4j-api-1.7.36.jar:lib/spymemcached-2.12.3.jar:lib/stax-ex-1.8.jar:lib/tesla-aether-0.0.5.jar:lib/txw2-2.3.1.jar:lib/validation-api-1.1.0.Final.jar:lib/vavr-0.10.2.jar:lib/vavr-match-0.10.2.jar:lib/wagon-provider-api-2.4.jar:lib/web-console-25.0.0.jar:lib/xz-1.8.jar:lib/zookeeper-3.5.9.jar:lib/zookeeper-jute-3.5.9.jar:lib/zstd-jni-1.5.2-3.jar: 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * java.class.version: 55.0 2023-02-09T16:53:17,797 INFO [main] org.apache.druid.cli.CliPeon - * java.home: /usr/lib/jvm/java-11-openjdk-amd64 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.io.tmpdir: /tmp 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.library.path: /usr/java/packages/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.runtime.name: OpenJDK Runtime Environment 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.runtime.version: 11.0.16+8-post-Debian-1deb11u1 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.specification.name: Java Platform API Specification 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.specification.vendor: Oracle Corporation 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.specification.version: 11 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.util.logging.manager: org.apache.logging.log4j.jul.LogManager 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.vendor: Debian 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.vendor.url: https://tracker.debian.org/openjdk-11 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.vendor.url.bug: https://bugs.debian.org/openjdk-11 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.version: 11.0.16 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.version.date: 2022-07-19 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.vm.compressedOopsMode: Zero based 2023-02-09T16:53:17,798 INFO [main] org.apache.druid.cli.CliPeon - * java.vm.info: mixed mode 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * java.vm.name: OpenJDK 64-Bit Server VM 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * java.vm.specification.name: Java Virtual Machine Specification 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * java.vm.specification.vendor: Oracle Corporation 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * java.vm.specification.version: 11 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * java.vm.vendor: Debian 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * java.vm.version: 11.0.16+8-post-Debian-1deb11u1 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * jdk.debug: release 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * line.separator: 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * log4j.shutdownCallbackRegistry: org.apache.druid.common.config.Log4jShutdown 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * log4j.shutdownHookEnabled: true 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * log4j2.is.webapp: false 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * os.arch: amd64 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * os.name: Linux 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * os.version: 5.14.0-70.30.1.el9_0.x86_64 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * path.separator: : 2023-02-09T16:53:17,799 INFO [main] org.apache.druid.cli.CliPeon - * sun.arch.data.model: 64 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * sun.boot.library.path: /usr/lib/jvm/java-11-openjdk-amd64/lib 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * sun.cpu.endian: little 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * sun.cpu.isalist: 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * sun.io.unicode.encoding: UnicodeLittle 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * sun.java.command: org.apache.druid.cli.Main internal peon var/druid/task/noop_2023-02-09T16:53:12.037Z_3c8e9f26-6ca4-4600-8723-572d2c8baf48 1 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * sun.java.launcher: SUN_STANDARD 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * sun.jnu.encoding: UTF-8 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * sun.management.compiler: HotSpot 64-Bit Tiered Compilers 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * sun.os.patch.level: unknown 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * user.dir: /opt/druid 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * user.home: /opt/druid 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * user.language: en 2023-02-09T16:53:17,800 INFO [main] org.apache.druid.cli.CliPeon - * user.name: druid 2023-02-09T16:53:17,801 INFO [main] org.apache.druid.cli.CliPeon - * user.timezone: UTC 2023-02-09T16:53:17,867 WARN [main] org.apache.druid.indexing.common.config.TaskConfig - Batch processing mode argument value is null or not valid:[null], defaulting to[CLOSED_SEGMENTS] 2023-02-09T16:53:18,667 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [druid.computation.buffer.poolCacheMaxCount, ${base_path}.buffer.poolCacheMaxCount] on [org.apache.druid.query.DruidProcessingConfig#poolCacheMaxCount()] 2023-02-09T16:53:18,668 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.merge.pool.awaitShutdownMillis] on [org.apache.druid.query.DruidProcessingConfig#getMergePoolAwaitShutdownMillis()] 2023-02-09T16:53:18,668 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [druid.computation.buffer.size, ${base_path}.buffer.sizeBytes] on [org.apache.druid.query.DruidProcessingConfig#intermediateComputeSizeBytesConfigured()] 2023-02-09T16:53:18,668 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [druid.computation.buffer.poolCacheInitialCount, ${base_path}.buffer.poolCacheInitialCount] on [org.apache.druid.query.DruidProcessingConfig#getNumInitalBuffersForIntermediatePool()] 2023-02-09T16:53:18,668 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.numThreads] on [org.apache.druid.query.DruidProcessingConfig#getNumThreadsConfigured()] 2023-02-09T16:53:18,668 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.numMergeBuffers] on [org.apache.druid.query.DruidProcessingConfig#getNumMergeBuffersConfigured()] 2023-02-09T16:53:18,668 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.columnCache.sizeBytes] on [org.apache.druid.query.DruidProcessingConfig#columnCacheSizeBytes()] 2023-02-09T16:53:18,669 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.fifo] on [org.apache.druid.query.DruidProcessingConfig#isFifo()] 2023-02-09T16:53:18,669 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.tmpDir] on [org.apache.druid.query.DruidProcessingConfig#getTmpDir()] 2023-02-09T16:53:18,669 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.merge.useParallelMergePool] on [org.apache.druid.query.DruidProcessingConfig#useParallelMergePoolConfigured()] 2023-02-09T16:53:18,669 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.merge.pool.parallelism] on [org.apache.druid.query.DruidProcessingConfig#getMergePoolParallelismConfigured()] 2023-02-09T16:53:18,669 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.merge.pool.defaultMaxQueryParallelism] on [org.apache.druid.query.DruidProcessingConfig#getMergePoolDefaultMaxQueryParallelism()] 2023-02-09T16:53:18,669 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.merge.task.targetRunTimeMillis] on [org.apache.druid.query.DruidProcessingConfig#getMergePoolTargetTaskRunTimeMillis()] 2023-02-09T16:53:18,669 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.merge.task.initialYieldNumRows] on [org.apache.druid.query.DruidProcessingConfig#getMergePoolTaskInitialYieldRows()] 2023-02-09T16:53:18,669 INFO [main] org.skife.config.ConfigurationObjectFactory - Using method itself for [${base_path}.merge.task.smallBatchNumRows] on [org.apache.druid.query.DruidProcessingConfig#getMergePoolSmallBatchRows()] 2023-02-09T16:53:18,669 INFO [main] org.skife.config.ConfigurationObjectFactory - Assigning default value [processing-%s] for [${base_path}.formatString] on [org.apache.druid.java.util.common.concurrent.ExecutorServiceConfig#getFormatString()] 2023-02-09T16:53:18,779 INFO [main] org.apache.druid.segment.loading.SegmentLocalCacheManager - Using storage location strategy: [LeastBytesUsedStorageLocationSelectorStrategy] 2023-02-09T16:53:18,876 INFO [main] org.apache.druid.guice.StorageNodeModule - Segment cache not configured on ServerType [indexer-executor]. It will not be assignable for segment placement 2023-02-09T16:53:18,960 INFO [main] org.eclipse.jetty.util.log - Logging initialized @5880ms to org.eclipse.jetty.util.log.Slf4jLog 2023-02-09T16:53:18,971 INFO [main] org.apache.druid.server.initialization.jetty.JettyServerModule - Creating http connector with port [8100] 2023-02-09T16:53:19,096 WARN [main] org.eclipse.jetty.server.handler.gzip.GzipHandler - minGzipSize of 0 is inefficient for short content, break even is size 23 2023-02-09T16:53:19,161 INFO [main] org.apache.druid.guice.DruidProcessingModule - Unable to determine max direct memory size. If druid.processing.buffer.sizeBytes is explicitly configured, then make sure to set -XX:MaxDirectMemorySize to at least "druid.processing.buffer.sizeBytes * (druid.processing.numMergeBuffers[2] + druid.processing.numThreads[1] + 1)", or else set -XX:MaxDirectMemorySize to at least 25% of maximum jvm heap size. 2023-02-09T16:53:19,162 INFO [main] org.apache.druid.query.DruidProcessingConfig - Using up to [2,075,901,952] bytes of direct memory for computation buffers. 2023-02-09T16:53:19,162 INFO [main] org.apache.druid.query.DruidProcessingConfig - Auto sizing buffers to [518,975,488] bytes each for [1] processing and [2] merge buffers. If you run out of direct memory, you may need to set these parameters explicitly using the guidelines at https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#processing-threads-buffers. 2023-02-09T16:53:19,163 INFO [main] org.apache.druid.offheap.OffheapBufferGenerator - Allocating new intermediate processing buffer[0] of size[518,975,488] 2023-02-09T16:53:19,282 INFO [main] org.apache.druid.guice.DruidProcessingModule - Unable to determine max direct memory size. If druid.processing.buffer.sizeBytes is explicitly configured, then make sure to set -XX:MaxDirectMemorySize to at least "druid.processing.buffer.sizeBytes * (druid.processing.numMergeBuffers[2] + druid.processing.numThreads[1] + 1)", or else set -XX:MaxDirectMemorySize to at least 25% of maximum jvm heap size. 2023-02-09T16:53:19,283 INFO [main] org.apache.druid.offheap.OffheapBufferGenerator - Allocating new result merging buffer[0] of size[518,975,488] 2023-02-09T16:53:19,381 INFO [main] org.apache.druid.offheap.OffheapBufferGenerator - Allocating new result merging buffer[1] of size[518,975,488] 2023-02-09T16:53:19,455 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Starting lifecycle [module] stage [INIT] 2023-02-09T16:53:19,455 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Starting lifecycle [module] stage [NORMAL] 2023-02-09T16:53:19,459 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider - starting 2023-02-09T16:53:19,459 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider - started 2023-02-09T16:53:19,460 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider - Creating NodeRoleWatcher for nodeRole [OVERLORD]. 2023-02-09T16:53:19,461 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Starting NodeRoleWatcher for [OVERLORD]... 2023-02-09T16:53:19,462 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Started NodeRoleWatcher for [OVERLORD]. 2023-02-09T16:53:19,462 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider - Created NodeRoleWatcher for nodeRole [OVERLORD]. 2023-02-09T16:53:19,462 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider - Creating NodeRoleWatcher for nodeRole [COORDINATOR]. 2023-02-09T16:53:19,462 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Starting NodeRoleWatcher for [COORDINATOR]... 2023-02-09T16:53:19,463 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Started NodeRoleWatcher for [COORDINATOR]. 2023-02-09T16:53:19,463 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider - Created NodeRoleWatcher for nodeRole [COORDINATOR]. 2023-02-09T16:53:19,476 INFO [main] org.apache.druid.indexing.worker.executor.ExecutorLifecycle - Running with task: { "type" : "noop", "id" : "noop_2023-02-09T16:53:12.037Z_3c8e9f26-6ca4-4600-8723-572d2c8baf48", "groupId" : "noop_2023-02-09T16:53:12.037Z_3c8e9f26-6ca4-4600-8723-572d2c8baf48", "dataSource" : "none", "runTime" : 2500, "isReadyTime" : 0, "isReadyResult" : "YES", "firehose" : null, "context" : { "forceTimeChunkLock" : true, "useLineageBasedSegmentAllocation" : true }, "resource" : { "availabilityGroup" : "noop_2023-02-09T16:53:12.037Z_3c8e9f26-6ca4-4600-8723-572d2c8baf48", "requiredCapacity" : 1 } } 2023-02-09T16:53:19,479 INFO [main] org.apache.druid.segment.loading.SegmentLocalCacheManager - Using storage location strategy: [LeastBytesUsedStorageLocationSelectorStrategy] 2023-02-09T16:53:19,482 INFO [task-runner-0-priority-0] org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner - Running task: noop_2023-02-09T16:53:12.037Z_3c8e9f26-6ca4-4600-8723-572d2c8baf48 2023-02-09T16:53:19,561 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Starting lifecycle [module] stage [SERVER] 2023-02-09T16:53:19,564 INFO [main] org.eclipse.jetty.server.Server - jetty-9.4.48.v20220622; built: 2022-06-21T20:42:25.880Z; git: 6b67c5719d1f4371b33655ff2d047d24e171e49a; jvm 11.0.16+8-post-Debian-1deb11u1 2023-02-09T16:53:19,802 INFO [main] org.eclipse.jetty.server.session - DefaultSessionIdManager workerName=node0 2023-02-09T16:53:19,802 INFO [main] org.eclipse.jetty.server.session - No SessionScavenger set, using defaults 2023-02-09T16:53:19,803 INFO [main] org.eclipse.jetty.server.session - node0 Scavenging every 660000ms 2023-02-09T16:53:20,062 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Registering com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider as a provider class 2023-02-09T16:53:20,063 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Registering com.fasterxml.jackson.jaxrs.smile.JacksonSmileProvider as a provider class 2023-02-09T16:53:20,063 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Registering org.apache.druid.server.initialization.jetty.CustomExceptionMapper as a provider class 2023-02-09T16:53:20,063 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Registering org.apache.druid.server.initialization.jetty.ForbiddenExceptionMapper as a provider class 2023-02-09T16:53:20,063 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Registering org.apache.druid.server.initialization.jetty.BadRequestExceptionMapper as a provider class 2023-02-09T16:53:20,063 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Registering org.apache.druid.server.initialization.jetty.ServiceUnavailableExceptionMapper as a provider class 2023-02-09T16:53:20,063 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Registering org.apache.druid.server.StatusResource as a root resource class 2023-02-09T16:53:20,065 INFO [main] com.sun.jersey.server.impl.application.WebApplicationImpl - Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2023-02-09T16:53:20,177 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.server.initialization.jetty.CustomExceptionMapper to GuiceManagedComponentProvider with the scope "Singleton" 2023-02-09T16:53:20,178 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.server.initialization.jetty.ForbiddenExceptionMapper to GuiceManagedComponentProvider with the scope "Singleton" 2023-02-09T16:53:20,178 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.server.initialization.jetty.BadRequestExceptionMapper to GuiceManagedComponentProvider with the scope "Singleton" 2023-02-09T16:53:20,178 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.server.initialization.jetty.ServiceUnavailableExceptionMapper to GuiceManagedComponentProvider with the scope "Singleton" 2023-02-09T16:53:20,254 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider to GuiceManagedComponentProvider with the scope "Singleton" 2023-02-09T16:53:20,259 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding com.fasterxml.jackson.jaxrs.smile.JacksonSmileProvider to GuiceManagedComponentProvider with the scope "Singleton" 2023-02-09T16:53:20,572 INFO [org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcheroverlord] org.apache.druid.discovery.BaseNodeRoleWatcher - Node watcher of role [overlord] is now initialized with 1 nodes. 2023-02-09T16:53:20,572 INFO [org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatchercoordinator] org.apache.druid.discovery.BaseNodeRoleWatcher - Node watcher of role [coordinator] is now initialized with 2 nodes. 2023-02-09T16:53:20,858 INFO [task-runner-0-priority-0] org.apache.druid.indexing.common.task.NoopTask - Running noop task[noop_2023-02-09T16:53:12.037Z_3c8e9f26-6ca4-4600-8723-572d2c8baf48] 2023-02-09T16:53:20,858 INFO [task-runner-0-priority-0] org.apache.druid.indexing.common.task.NoopTask - Sleeping for 2,500 millis. 2023-02-09T16:53:20,892 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.server.http.security.StateResourceFilter to GuiceInstantiatedComponentProvider 2023-02-09T16:53:20,968 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.server.http.SegmentListerResource to GuiceInstantiatedComponentProvider 2023-02-09T16:53:20,973 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.server.QueryResource to GuiceInstantiatedComponentProvider 2023-02-09T16:53:20,973 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.segment.realtime.firehose.ChatHandlerResource to GuiceInstantiatedComponentProvider 2023-02-09T16:53:20,975 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.server.http.security.ConfigResourceFilter to GuiceInstantiatedComponentProvider 2023-02-09T16:53:20,977 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.query.lookup.LookupListeningResource to GuiceInstantiatedComponentProvider 2023-02-09T16:53:20,978 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.query.lookup.LookupIntrospectionResource to GuiceInstantiatedComponentProvider 2023-02-09T16:53:20,979 INFO [main] com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory - Binding org.apache.druid.server.StatusResource to GuiceManagedComponentProvider with the scope "Undefined" 2023-02-09T16:53:20,995 INFO [main] org.eclipse.jetty.server.handler.ContextHandler - Started o.e.j.s.ServletContextHandler@608c7b5b{/,null,AVAILABLE} 2023-02-09T16:53:21,057 INFO [main] org.eclipse.jetty.server.AbstractConnector - Started ServerConnector@5a5b394{HTTP/1.1, (http/1.1)}{0.0.0.0:8100} 2023-02-09T16:53:21,057 INFO [main] org.eclipse.jetty.server.Server - Started @7978ms 2023-02-09T16:53:21,057 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Starting lifecycle [module] stage [ANNOUNCEMENTS] 2023-02-09T16:53:21,057 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Successfully started lifecycle [module] 2023-02-09T16:53:23,358 INFO [task-runner-0-priority-0] org.apache.druid.indexing.common.task.NoopTask - Woke up! 2023-02-09T16:53:23,371 INFO [task-runner-0-priority-0] org.apache.druid.indexing.worker.executor.ExecutorLifecycle - Task completed with status: { "id" : "noop_2023-02-09T16:53:12.037Z_3c8e9f26-6ca4-4600-8723-572d2c8baf48", "status" : "SUCCESS", "duration" : 3887, "errorMsg" : null, "location" : { "host" : null, "port" : -1, "tlsPort" : -1 } } 2023-02-09T16:53:23,375 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Stopping lifecycle [module] stage [ANNOUNCEMENTS] 2023-02-09T16:53:23,376 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Stopping lifecycle [module] stage [SERVER] 2023-02-09T16:53:23,380 INFO [main] org.eclipse.jetty.server.AbstractConnector - Stopped ServerConnector@5a5b394{HTTP/1.1, (http/1.1)}{0.0.0.0:8100} 2023-02-09T16:53:23,380 INFO [main] org.eclipse.jetty.server.session - node0 Stopped scavenging 2023-02-09T16:53:23,381 INFO [main] org.eclipse.jetty.server.handler.ContextHandler - Stopped o.e.j.s.ServletContextHandler@608c7b5b{/,null,STOPPED} 2023-02-09T16:53:23,385 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Stopping lifecycle [module] stage [NORMAL] 2023-02-09T16:53:23,385 INFO [main] org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner - Starting graceful shutdown of task[noop_2023-02-09T16:53:12.037Z_3c8e9f26-6ca4-4600-8723-572d2c8baf48]. 2023-02-09T16:53:23,388 INFO [LookupExtractorFactoryContainerProvider-MainThread] org.apache.druid.query.lookup.LookupReferencesManager - Lookup Management loop exited. Lookup notices are not handled anymore. 2023-02-09T16:53:23,388 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider - stopping 2023-02-09T16:53:23,388 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Stopping NodeRoleWatcher for [OVERLORD]... 2023-02-09T16:53:30,658 ERROR [org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcheroverlord] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Expection while watching for NodeRole [OVERLORD]. java.lang.RuntimeException: java.lang.InterruptedException at org.apache.druid.concurrent.LifecycleLock$Sync.awaitStarted(LifecycleLock.java:144) ~[druid-core-25.0.0.jar:25.0.0] at org.apache.druid.concurrent.LifecycleLock.awaitStarted(LifecycleLock.java:245) ~[druid-core-25.0.0.jar:25.0.0] at org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher.keepWatching(K8sDruidNodeDiscoveryProvider.java:257) ~[?:?] at org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher.watch(K8sDruidNodeDiscoveryProvider.java:237) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:829) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1367) ~[?:?] at org.apache.druid.concurrent.LifecycleLock$Sync.awaitStarted(LifecycleLock.java:139) ~[druid-core-25.0.0.jar:25.0.0] ... 8 more 2023-02-09T16:53:38,389 WARN [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Failed to stop watchExecutor for NodeRoleWatcher[OVERLORD] 2023-02-09T16:53:38,389 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Stopped NodeRoleWatcher for [OVERLORD]. 2023-02-09T16:53:38,389 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Stopping NodeRoleWatcher for [COORDINATOR]... 2023-02-09T16:53:38,390 ERROR [org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatchercoordinator] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Expection while watching for NodeRole [COORDINATOR]. java.lang.RuntimeException: java.lang.InterruptedException at org.apache.druid.concurrent.LifecycleLock$Sync.awaitStarted(LifecycleLock.java:144) ~[druid-core-25.0.0.jar:25.0.0] at org.apache.druid.concurrent.LifecycleLock.awaitStarted(LifecycleLock.java:245) ~[druid-core-25.0.0.jar:25.0.0] at org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher.keepWatching(K8sDruidNodeDiscoveryProvider.java:257) ~[?:?] at org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher.watch(K8sDruidNodeDiscoveryProvider.java:237) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:829) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1367) ~[?:?] at org.apache.druid.concurrent.LifecycleLock$Sync.awaitStarted(LifecycleLock.java:139) ~[druid-core-25.0.0.jar:25.0.0] ... 8 more 2023-02-09T16:53:40,660 INFO [org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcheroverlord] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Exited Watch for NodeRole [OVERLORD]. 2023-02-09T16:53:48,391 INFO [org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatchercoordinator] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Exited Watch for NodeRole [COORDINATOR]. 2023-02-09T16:53:48,391 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider$NodeRoleWatcher - Stopped NodeRoleWatcher for [COORDINATOR]. 2023-02-09T16:53:48,391 INFO [main] org.apache.druid.k8s.discovery.K8sDruidNodeDiscoveryProvider - stopped 2023-02-09T16:53:48,408 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Stopping lifecycle [module] stage [INIT] Finished peon task