2023-12-19 17:03:19,400 [pool-13-thread-2] INFO c.l.m.filter.RestliLoggingFilter:55 - POST /entities?action=ingest - ingest - 200 - 5984ms 2023-12-19 17:03:19,533 [qtp496729294-896] INFO c.l.m.filter.RestliLoggingFilter:55 - POST /entities?action=ingest - ingest - 500 - 13345ms 2023-12-19 17:03:19,622 [qtp496729294-896] ERROR c.l.m.filter.RestliLoggingFilter:38 - Rest.li error: com.linkedin.restli.server.RestLiServiceException: java.lang.RuntimeException: Failed to produce MCLs at com.linkedin.metadata.restli.RestliUtil.toTask(RestliUtil.java:42) at com.linkedin.metadata.restli.RestliUtil.toTask(RestliUtil.java:50) at com.linkedin.metadata.resources.entity.EntityResource.ingest(EntityResource.java:262) at jdk.internal.reflect.GeneratedMethodAccessor187.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.linkedin.restli.internal.server.RestLiMethodInvoker.doInvoke(RestLiMethodInvoker.java:177) at com.linkedin.restli.internal.server.RestLiMethodInvoker.invoke(RestLiMethodInvoker.java:333) at com.linkedin.restli.internal.server.filter.FilterChainDispatcherImpl.onRequestSuccess(FilterChainDispatcherImpl.java:47) at com.linkedin.restli.internal.server.filter.RestLiFilterChainIterator.onRequest(RestLiFilterChainIterator.java:86) at com.linkedin.restli.internal.server.filter.RestLiFilterChainIterator.lambda$onRequest$0(RestLiFilterChainIterator.java:73) at java.base/java.util.concurrent.CompletableFuture.uniAcceptNow(CompletableFuture.java:753) at java.base/java.util.concurrent.CompletableFuture.uniAcceptStage(CompletableFuture.java:731) at java.base/java.util.concurrent.CompletableFuture.thenAccept(CompletableFuture.java:2108) at com.linkedin.restli.internal.server.filter.RestLiFilterChainIterator.onRequest(RestLiFilterChainIterator.java:72) at com.linkedin.restli.internal.server.filter.RestLiFilterChain.onRequest(RestLiFilterChain.java:55) at com.linkedin.restli.server.BaseRestLiServer.handleResourceRequest(BaseRestLiServer.java:270) at com.linkedin.restli.server.RestRestLiServer.handleResourceRequestWithRestLiResponse(RestRestLiServer.java:294) at com.linkedin.restli.server.RestRestLiServer.handleResourceRequest(RestRestLiServer.java:262) at com.linkedin.restli.server.RestRestLiServer.handleResourceRequest(RestRestLiServer.java:232) at com.linkedin.restli.server.RestRestLiServer.doHandleRequest(RestRestLiServer.java:215) at com.linkedin.restli.server.RestRestLiServer.handleRequest(RestRestLiServer.java:171) at com.linkedin.restli.server.RestLiServer.handleRequest(RestLiServer.java:132) at com.linkedin.restli.server.DelegatingTransportDispatcher.handleRestRequest(DelegatingTransportDispatcher.java:70) at com.linkedin.r2.filter.transport.DispatcherRequestFilter.onRestRequest(DispatcherRequestFilter.java:70) at com.linkedin.r2.filter.TimedRestFilter.onRestRequest(TimedRestFilter.java:76) at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:146) at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:132) at com.linkedin.r2.filter.FilterChainIterator.onRequest(FilterChainIterator.java:62) at com.linkedin.r2.filter.TimedNextFilter.onRequest(TimedNextFilter.java:55) at com.linkedin.r2.filter.transport.ServerQueryTunnelFilter.onRestRequest(ServerQueryTunnelFilter.java:58) at com.linkedin.r2.filter.TimedRestFilter.onRestRequest(TimedRestFilter.java:76) at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:146) at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:132) at com.linkedin.r2.filter.FilterChainIterator.onRequest(FilterChainIterator.java:62) at com.linkedin.r2.filter.TimedNextFilter.onRequest(TimedNextFilter.java:55) at com.linkedin.r2.filter.message.rest.RestFilter.onRestRequest(RestFilter.java:50) at com.linkedin.r2.filter.TimedRestFilter.onRestRequest(TimedRestFilter.java:76) at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:146) at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:132) at com.linkedin.r2.filter.FilterChainIterator.onRequest(FilterChainIterator.java:62) at com.linkedin.r2.filter.FilterChainImpl.onRestRequest(FilterChainImpl.java:106) at com.linkedin.r2.filter.transport.FilterChainDispatcher.handleRestRequest(FilterChainDispatcher.java:75) at com.linkedin.r2.util.finalizer.RequestFinalizerDispatcher.handleRestRequest(RequestFinalizerDispatcher.java:61) at com.linkedin.r2.transport.http.server.HttpDispatcher.handleRequest(HttpDispatcher.java:101) at com.linkedin.r2.transport.http.server.AbstractR2Servlet.service(AbstractR2Servlet.java:105) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at com.linkedin.restli.server.RestliHandlerServlet.service(RestliHandlerServlet.java:21) at com.linkedin.restli.server.RestliHandlerServlet.handleRequest(RestliHandlerServlet.java:26) at org.springframework.web.context.support.HttpRequestHandlerServlet.service(HttpRequestHandlerServlet.java:73) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799) at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1631) at com.datahub.auth.authentication.filter.AuthenticationFilter.doFilter(AuthenticationFilter.java:102) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:600) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1440) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1355) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:191) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: java.lang.RuntimeException: Failed to produce MCLs at com.linkedin.metadata.entity.EntityServiceImpl.emitMCL(EntityServiceImpl.java:684) at com.linkedin.metadata.entity.EntityServiceImpl.ingestAspects(EntityServiceImpl.java:543) at com.linkedin.metadata.entity.EntityServiceImpl.ingestSnapshotUnion(EntityServiceImpl.java:1314) at com.linkedin.metadata.entity.EntityServiceImpl.ingestEntity(EntityServiceImpl.java:1165) at com.linkedin.metadata.resources.entity.EntityResource.lambda$ingest$5(EntityResource.java:263) at com.linkedin.metadata.restli.RestliUtil.toTask(RestliUtil.java:30) ... 88 common frames omitted 2023-12-19 17:03:19,670 [qtp496729294-1089] INFO c.l.m.entity.EntityServiceImpl:1301 - INGEST urn urn:li:dataset:(urn:li:dataPlatform:looker,shiphero.explore.workspace_member_onboarding_survey,PROD) with system metadata {lastObserved=1703005375897, runId=ec9b12f6-2e5e-4295-a9ac-7564f2edc96f} 2023-12-19 17:03:20,482 [qtp496729294-896] INFO c.l.m.entity.EntityServiceImpl:1301 - INGEST urn urn:li:dataset:(urn:li:dataPlatform:looker,shiphero.explore.fulfillment_members,PROD) with system metadata {lastObserved=1703005372499, runId=ec9b12f6-2e5e-4295-a9ac-7564f2edc96f} 2023-12-19 17:03:20,518 [Thread-985] WARN org.elasticsearch.client.RestClient:65 - request [POST http://elasticsearch-master:9200/datahubingestionsourceindex_v2/_search?typed_keys=true&max_concurrent_shard_requests=5&ignore_unavailable=false&expand_wildcards=open&allow_no_indices=true&ignore_throttled=true&search_type=query_then_fetch&batched_reduce_size=512&ccs_minimize_roundtrips=true] returned 2 warnings: [299 Elasticsearch-7.17.3-5ad023604c8d7416c9eb6c0eadb62b14e766caff "Elasticsearch built-in security features are not enabled. Without authentication, your cluster could be accessible to anyone. See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/security-minimal-setup.html to enable security."],[299 Elasticsearch-7.17.3-5ad023604c8d7416c9eb6c0eadb62b14e766caff "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices."]