Stuck on an issue?

Lightrun Answers was designed to reduce the constant googling that comes with debugging 3rd party libraries. It collects links to all the places you might be looking at while hunting down a tough bug.

And, if you’re still stuck at the end, we’re happy to hop on a call to see how we can help out.

upgrade to 24.0.0 from 0.23.0, Hadoop index task fail

See original GitHub issue

Please provide a detailed title (e.g. “Broker crashes when using TopN query with Bound filter” instead of just “Broker crashes”).

Affected Version

24.0.0

Description

upgrade from 0.23.0, hadoop index task fail.

_common/runtime.properties

druid.extensions.loadList=["druid-datasketches", "druid-lookups-cached-global", "mysql-metadata-storage", "druid-hdfs-storage", "druid-multi-stage-query"]

payload

{
  "type": "index_hadoop",
  "id": "index_yzjconnectapp@event_2022-10-28T11:00_2022-10-28T12:00_2022-10-28T16:25",
  "spec": {
    "dataSchema": {
      "dataSource": "yzjconnectapp@event",
      "timestampSpec": null,
      "dimensionsSpec": null,
      "metricsSpec": [],
      "granularitySpec": {
        "type": "uniform",
        "segmentGranularity": "HOUR",
        "queryGranularity": "HOUR",
        "rollup": false,
        "intervals": [
          "2022-10-28T03:00:00.000Z/2022-10-28T04:00:00.000Z"
        ]
      },
      "transformSpec": {
        "filter": null,
        "transforms": []
      },
      "parser": {
        "type": "hadoopyString",
        "parseSpec": {
          "format": "json",
          "dimensionsSpec": {
            "dimensions": [
              "event",
              "eid",
              "oid",
              "spaceName",
              "spaceId",
              "ename",
              "appNum",
              "mainUserFlag",
              "userName",
              "userRole",
              "appType",
              "appName",
              "partnerSum",
              "@network_provider",
              "@country",
              "@city",
              "@province",
              "@user_agent",
              "@os",
              "@os_version",
              "@clientId",
              "@deviceid"
            ]
          },
          "timestampSpec": {
            "format": "auto",
            "column": "@timestamp"
          }
        }
      }
    },
    "ioConfig": {
      "type": "hadoop",
      "inputSpec": {
        "type": "static",
        "paths": "hdfs://hadoop.architecture:8020/xxxx/dataly/druid/history/yzjconnectapp@event/2022-10-28_11.log"
      },
      "metadataUpdateSpec": null,
      "segmentOutputPath": null
    },
    "tuningConfig": {
      "type": "hadoop",
      "workingPath": null,
      "version": "2022-10-28T08:25:20.425Z",
      "partitionsSpec": {
        "type": "hashed",
        "numShards": null,
        "partitionDimensions": [],
        "partitionFunction": "murmur3_32_abs",
        "maxRowsPerSegment": 6000000
      },
      "shardSpecs": {},
      "indexSpec": {
        "bitmap": {
          "type": "roaring",
          "compressRunOnSerialization": true
        },
        "dimensionCompression": "lz4",
        "metricCompression": "lz4",
        "longEncoding": "longs",
        "segmentLoader": null
      },
      "indexSpecForIntermediatePersists": {
        "bitmap": {
          "type": "roaring",
          "compressRunOnSerialization": true
        },
        "dimensionCompression": "lz4",
        "metricCompression": "lz4",
        "longEncoding": "longs",
        "segmentLoader": null
      },
      "appendableIndexSpec": {
        "type": "onheap",
        "preserveExistingMetrics": false
      },
      "maxRowsInMemory": 1000000,
      "maxBytesInMemory": 0,
      "leaveIntermediate": false,
      "cleanupOnFailure": true,
      "overwriteFiles": false,
      "ignoreInvalidRows": true,
      "jobProperties": {
        "mapreduce.map.java.opts": "-Duser.timezone=UTC+8 -Dfile.encoding=UTF-8",
        "mapreduce.reduce.java.opts": "-Duser.timezone=UTC+8 -Dfile.encoding=UTF-8",
        "mapreduce.cluster.local.dir": "${hadoop.tmp.dir}/mapred/local/index_yzjconnectapp@event_2022-10-28T11:00_2022-10-28T12:00_2022-10-28T16:25"
      },
      "combineText": false,
      "useCombiner": false,
      "numBackgroundPersistThreads": 0,
      "forceExtendableShardSpecs": false,
      "useExplicitVersion": false,
      "allowedHadoopPrefix": [],
      "logParseExceptions": false,
      "maxParseExceptions": 2147483647,
      "useYarnRMJobStatusFallback": true,
      "awaitSegmentAvailabilityTimeoutMillis": 0
    },
    "uniqueId": "03b57b35798e481d9b005051a481f1c9"
  },
  "hadoopDependencyCoordinates": null,
  "classpathPrefix": null,
  "context": {
    "forceTimeChunkLock": true,
    "useLineageBasedSegmentAllocation": true
  },
  "groupId": "index_yzjconnectapp@event_2022-10-28T11:00_2022-10-28T12:00_2022-10-28T16:25",
  "dataSource": "yzjconnectapp@event",
  "resource": {
    "availabilityGroup": "index_yzjconnectapp@event_2022-10-28T11:00_2022-10-28T12:00_2022-10-28T16:25",
    "requiredCapacity": 1
  }
}

2022-10-28T16:25:34,775 ERROR [task-runner-0-priority-0] org.apache.druid.indexing.common.task.HadoopIndexTask - Got invocation target exception in run()
java.lang.ExceptionInInitializerError: null
	at org.apache.druid.indexing.common.task.HadoopIndexTask$HadoopDetermineConfigInnerProcessingRunner.runTask(HadoopIndexTask.java:780) ~[druid-indexing-service-24.0.0.jar:24.0.0]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_342]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_342]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_342]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_342]
	at org.apache.druid.indexing.common.task.HadoopIndexTask.runInternal(HadoopIndexTask.java:366) ~[druid-indexing-service-24.0.0.jar:24.0.0]
	at org.apache.druid.indexing.common.task.HadoopIndexTask.runTask(HadoopIndexTask.java:297) ~[druid-indexing-service-24.0.0.jar:24.0.0]
	at org.apache.druid.indexing.common.task.AbstractBatchIndexTask.run(AbstractBatchIndexTask.java:187) ~[druid-indexing-service-24.0.0.jar:24.0.0]
	at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:477) ~[druid-indexing-service-24.0.0.jar:24.0.0]
	at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:449) ~[druid-indexing-service-24.0.0.jar:24.0.0]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_342]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_342]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_342]
	at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_342]
Caused by: com.google.inject.CreationException: Unable to create injector, see the following errors:

1) No implementation for java.util.Set<org.apache.druid.discovery.NodeRole> annotated with @org.apache.druid.guice.annotations.Self() was bound.
  while locating java.util.Set<org.apache.druid.discovery.NodeRole> annotated with @org.apache.druid.guice.annotations.Self()
    for the 2nd parameter of org.apache.druid.msq.guice.MSQIndexingModule.makeBouncer(MSQIndexingModule.java:193)
  at org.apache.druid.msq.guice.MSQIndexingModule.makeBouncer(MSQIndexingModule.java:193) (via modules: com.google.inject.util.Modules$OverrideModule -> org.apache.druid.msq.guice.MSQIndexingModule)

2) A binding to com.google.common.base.Supplier<org.apache.druid.server.audit.SQLAuditManagerConfig> was already configured at org.apache.druid.guice.JsonConfigProvider.bind(JsonConfigProvider.java:151) (via modules: com.google.inject.util.Modules$OverrideModule -> org.apache.druid.metadata.storage.mysql.MySQLMetadataStorageModule).
  at org.apache.druid.guice.JsonConfigProvider.bind(JsonConfigProvider.java:151) (via modules: com.google.inject.util.Modules$OverrideModule -> org.apache.druid.metadata.storage.postgresql.PostgreSQLMetadataStorageModule)

3) A binding to org.apache.druid.server.audit.SQLAuditManagerConfig was already configured at org.apache.druid.guice.JsonConfigProvider.bind(JsonConfigProvider.java:152) (via modules: com.google.inject.util.Modules$OverrideModule -> org.apache.druid.metadata.storage.mysql.MySQLMetadataStorageModule).
  at org.apache.druid.guice.JsonConfigProvider.bind(JsonConfigProvider.java:152) (via modules: com.google.inject.util.Modules$OverrideModule -> org.apache.druid.metadata.storage.postgresql.PostgreSQLMetadataStorageModule)

4) No implementation for java.util.Set<org.apache.druid.discovery.NodeRole> annotated with @org.apache.druid.guice.annotations.Self() was bound.
  at org.apache.druid.msq.guice.MSQIndexingModule.makeBouncer(MSQIndexingModule.java:193) (via modules: com.google.inject.util.Modules$OverrideModule -> org.apache.druid.msq.guice.MSQIndexingModule)

4 errors
	at com.google.inject.internal.Errors.throwCreationExceptionIfErrorsExist(Errors.java:470) ~[guice-4.1.0.jar:?]
	at com.google.inject.internal.InternalInjectorCreator.initializeStatically(InternalInjectorCreator.java:155) ~[guice-4.1.0.jar:?]
	at com.google.inject.internal.InternalInjectorCreator.build(InternalInjectorCreator.java:107) ~[guice-4.1.0.jar:?]
	at com.google.inject.Guice.createInjector(Guice.java:99) ~[guice-4.1.0.jar:?]
	at com.google.inject.Guice.createInjector(Guice.java:73) ~[guice-4.1.0.jar:?]
	at com.google.inject.Guice.createInjector(Guice.java:62) ~[guice-4.1.0.jar:?]
	at org.apache.druid.initialization.ExtensionInjectorBuilder.build(ExtensionInjectorBuilder.java:49) ~[druid-server-24.0.0.jar:24.0.0]
	at org.apache.druid.initialization.ServerInjectorBuilder.build(ServerInjectorBuilder.java:113) ~[druid-server-24.0.0.jar:24.0.0]
	at org.apache.druid.initialization.ServerInjectorBuilder.makeServerInjector(ServerInjectorBuilder.java:71) ~[druid-server-24.0.0.jar:24.0.0]
	at org.apache.druid.initialization.Initialization.makeInjectorWithModules(Initialization.java:63) ~[druid-server-24.0.0.jar:24.0.0]
	at org.apache.druid.indexer.HadoopDruidIndexerConfig.<clinit>(HadoopDruidIndexerConfig.java:109) ~[druid-indexing-hadoop-24.0.0.jar:24.0.0]
	... 14 more
2022-10-28T16:25:34,806 INFO [task-runner-0-priority-0] org.apache.druid.indexing.worker.executor.ExecutorLifecycle - Task completed with status: {
  "id" : "index_yzjconnectapp@event_2022-10-28T11:00_2022-10-28T12:00_2022-10-28T16:25",
  "status" : "FAILED",
  "duration" : 5052,
  "errorMsg" : "java.lang.ExceptionInInitializerError\n\tat org.apache.druid.indexing.common.task.HadoopIndexTask$Hado...",
  "location" : {
    "host" : null,
    "port" : -1,
    "tlsPort" : -1
  }
}

Issue Analytics

State:
Created a year ago
Comments:11 (10 by maintainers)

Top GitHub Comments

1reaction

abhishekagarwal87commented, Nov 23, 2022

will be fixed in 25.0 via https://github.com/apache/druid/pull/13371. My fix was only a partial fix.

0reactions

599166320commented, Nov 17, 2022


2022-11-17T12:21:09,346 INFO [task-runner-0-priority-0] org.apache.druid.indexing.common.task.HadoopIndexTask - No need for cleanup as index generator job did not even run
2022-11-17T12:21:09,346 ERROR [task-runner-0-priority-0] org.apache.druid.indexing.common.task.HadoopIndexTask - Got invocation target exception in run()
java.lang.ExceptionInInitializerError: null
        at org.apache.druid.indexing.common.task.HadoopIndexTask$HadoopDetermineConfigInnerProcessingRunner.runTask(HadoopIndexTask.java:781) ~[druid-indexing-service-24.0.1.jar:24.0.1]
        at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
        at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?]
        at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
        at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
        at org.apache.druid.indexing.common.task.HadoopIndexTask.runInternal(HadoopIndexTask.java:367) ~[druid-indexing-service-24.0.1.jar:24.0.1]
        at org.apache.druid.indexing.common.task.HadoopIndexTask.runTask(HadoopIndexTask.java:297) ~[druid-indexing-service-24.0.1.jar:24.0.1]
        at org.apache.druid.indexing.common.task.AbstractBatchIndexTask.run(AbstractBatchIndexTask.java:187) ~[druid-indexing-service-24.0.1.jar:24.0.1]
        at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:477) ~[druid-indexing-service-24.0.1.jar:24.0.1]
        at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:449) ~[druid-indexing-service-24.0.1.jar:24.0.1]
        at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
        at java.lang.Thread.run(Thread.java:834) ~[?:?]
Caused by: com.google.inject.CreationException: Unable to create injector, see the following errors:

1) No implementation for java.util.Set<org.apache.druid.discovery.NodeRole> annotated with @org.apache.druid.guice.annotations.Self() was bound.
  while locating java.util.Set<org.apache.druid.discovery.NodeRole> annotated with @org.apache.druid.guice.annotations.Self()
    for the 2nd parameter of org.apache.druid.server.metrics.WorkerTaskCountStatsMonitor.<init>(WorkerTaskCountStatsMonitor.java:44)
  while locating org.apache.druid.server.metrics.WorkerTaskCountStatsMonitor
  at org.apache.druid.server.metrics.MetricsModule.getMonitorScheduler(MetricsModule.java:104) (via modules: com.google.inject.util.Modules$OverrideModule -> com.google.inject.util.Modules$OverrideModule -> org.apache.druid.server.metrics.MetricsModule)
  at org.apache.druid.server.metrics.MetricsModule.getMonitorScheduler(MetricsModule.java:104) (via modules: com.google.inject.util.Modules$OverrideModule -> com.google.inject.util.Modules$OverrideModule -> org.apache.druid.server.metrics.MetricsModule)
  while locating org.apache.druid.java.util.metrics.MonitorScheduler
  at org.apache.druid.server.metrics.MetricsModule.configure(MetricsModule.java:89) (via modules: com.google.inject.util.Modules$OverrideModule -> com.google.inject.util.Modules$OverrideModule -> org.apache.druid.server.metrics.MetricsModule)
  while locating org.apache.druid.java.util.metrics.MonitorScheduler annotated with @com.google.inject.name.Named(value=ForTheEagerness)

1 error
        at com.google.inject.internal.Errors.throwCreationExceptionIfErrorsExist(Errors.java:470) ~[guice-4.1.0

Top Results From Across the Web

Fix the Injector creation in HadoopTask #13138 - GitHub

HadoopTask uses the injector to create an instance of ExtensionsLoader which contains the ... upgrade to 24.0.0 from 0.23.0, Hadoop index task fail...

Middlemanagers keep disappearing - Operations - Druid Forum

Relates to Apache Druid 24.0.0 Setup Hi. I'm currently evaluating running Druid on a Kubernetes Cluster instead of bare metal.

Upgrading Execution Engine for Apache Hadoop to a newer ...

If Execution Engine for Apache Hadoop is already installed on top of Cloud Pak for Data Version ... If you try to change...

docs/development/extensions-core/kinesis-ingestion.md - Gitee

These indexing tasks read events using Kinesis' own shard and sequence number mechanism to guarantee exactly-once ingestion. The supervisor oversees the state ...

Hadoop-based ingestion - Apache Druid

If the Hadoop jobs created by the indexing task are unable to retrieve their completion status from the JobHistory server, and this parameter...