== Unoptimized Logical Plan ==

* Filter: col(__er_match_score) >= lit(1.0)
|
* Project: col(__store_cluster_id_l), col(__er_entity_pattern_id_l), col(__store_cluster_id_r), col(__er_entity_pattern_id_r), cast(lit(1) as Float64) as __er_match_score
|
* Filter: col(__er_entity_pattern_id_l) < col(__er_entity_pattern_id_r)
|
* Join: Type = Inner
|   Strategy = Hash
|   On = col(left.__store_cluster_id_l#String) == col(right.__store_cluster_id_r#String)
|   Output schema = __store_cluster_id_l#Utf8, __er_entity_pattern_id_l#Int64, __store_cluster_id_r#Utf8, __er_entity_pattern_id_r#Int64
|\
| * Repartition: Scheme = Hash
| |   Num partitions = Some(200)
| |   By = col(__store_cluster_id_r)
| |
| * Project: col(__store_cluster_id) as __store_cluster_id_r, col(__er_entity_pattern_id) as __er_entity_pattern_id_r
| |
| * Project: col(__store_cluster_id), col(__er_entity_pattern_id)
| |
| * GlobScanOperator
| |   Glob paths = [/tmp/entity_patterns]
| |   Coerce int96 timestamp unit = ns
| |   IO config = S3 config = { Max connections = 8, Retry initial backoff ms = 1000, Connect timeout ms = 30000, Read timeout ms = 30000, Max retries = 25, Retry mode = adaptive, Anonymous = false, Use SSL = true, Verify SSL = true, Check hostname SSL = true, Requester pays = false, Force Virtual Addressing = false }, Azure config = { Use
| |     Fabric Endpoint = false, Anonymous = false, Use SSL = true }, GCS config = { Anonymous = false, Max connections = 8, Retry initial backoff ms = 1000, Connect timeout ms = 30000, Read timeout ms = 30000, Max retries = 5 }, HTTP config = { User agent = daft/0.0.1, Retry initial backoff ms = 1000, Connect timeout ms = 30000, Read timeout ms =
| |     30000, Max retries = 5 }, Unity config = {  }, Hugging Face config = { Anonymous = false }, TOS config = { Anonymous = false, Max retries = 3, Retry timeout = 30000ms, Connect timeout = 10000ms, Read timeout = 30000ms, Max concurrent requests = 50, Max connections = 50 }
| |   Use multithreading = true
| |   File schema = __store_cluster_id#Utf8, __er_records#List(Struct([Field { name: "__er_source_name", dtype: Utf8, metadata: {} }, Field { name: "__er_source_id", dtype: Utf8, metadata: {} }, Field { name: "__er_record_id", dtype: Utf8, metadata: {} }])), __er_entity_pattern_id#Int64
| |   Partitioning keys = []
| |   Output schema = __store_cluster_id#Utf8, __er_records#List(Struct([Field { name: "__er_source_name", dtype: Utf8, metadata: {} }, Field { name: "__er_source_id", dtype: Utf8, metadata: {} }, Field { name: "__er_record_id", dtype: Utf8, metadata: {} }])), __er_entity_pattern_id#Int64
|
* Repartition: Scheme = Hash
|   Num partitions = Some(200)
|   By = col(__store_cluster_id_l)
|
* Project: col(__store_cluster_id) as __store_cluster_id_l, col(__er_entity_pattern_id) as __er_entity_pattern_id_l
|
* Project: col(__store_cluster_id), col(__er_entity_pattern_id)
|
* GlobScanOperator
|   Glob paths = [/tmp/entity_patterns]
|   Coerce int96 timestamp unit = ns
|   IO config = S3 config = { Max connections = 8, Retry initial backoff ms = 1000, Connect timeout ms = 30000, Read timeout ms = 30000, Max retries = 25, Retry mode = adaptive, Anonymous = false, Use SSL = true, Verify SSL = true, Check hostname SSL = true, Requester pays = false, Force Virtual Addressing = false }, Azure config = { Use Fabric
|     Endpoint = false, Anonymous = false, Use SSL = true }, GCS config = { Anonymous = false, Max connections = 8, Retry initial backoff ms = 1000, Connect timeout ms = 30000, Read timeout ms = 30000, Max retries = 5 }, HTTP config = { User agent = daft/0.0.1, Retry initial backoff ms = 1000, Connect timeout ms = 30000, Read timeout ms = 30000,
|     Max retries = 5 }, Unity config = {  }, Hugging Face config = { Anonymous = false }, TOS config = { Anonymous = false, Max retries = 3, Retry timeout = 30000ms, Connect timeout = 10000ms, Read timeout = 30000ms, Max concurrent requests = 50, Max connections = 50 }
|   Use multithreading = true
|   File schema = __store_cluster_id#Utf8, __er_records#List(Struct([Field { name: "__er_source_name", dtype: Utf8, metadata: {} }, Field { name: "__er_source_id", dtype: Utf8, metadata: {} }, Field { name: "__er_record_id", dtype: Utf8, metadata: {} }])), __er_entity_pattern_id#Int64
|   Partitioning keys = []
|   Output schema = __store_cluster_id#Utf8, __er_records#List(Struct([Field { name: "__er_source_name", dtype: Utf8, metadata: {} }, Field { name: "__er_source_id", dtype: Utf8, metadata: {} }, Field { name: "__er_record_id", dtype: Utf8, metadata: {} }])), __er_entity_pattern_id#Int64


== Optimized Logical Plan ==

* Filter: col(__er_match_score) >= lit(1.0)
|   Stats = { Approx num rows = 271,953, Approx size bytes = 7.33 MiB, Accumulated selectivity = 0.08 }
|
* Project: col(__store_cluster_id_l), col(__er_entity_pattern_id_l), col(__store_cluster_id_r), col(__er_entity_pattern_id_r), cast(lit(1) as Float64) as __er_match_score
|   Stats = { Approx num rows = 906,508, Approx size bytes = 24.42 MiB, Accumulated selectivity = 0.27 }
|
* Filter: col(__er_entity_pattern_id_l) < col(__er_entity_pattern_id_r)
|   Stats = { Approx num rows = 906,508, Approx size bytes = 24.42 MiB, Accumulated selectivity = 0.27 }
|
* Join: Type = Inner
|   Strategy = Hash
|   On = col(left.__store_cluster_id_l#String) == col(right.__store_cluster_id_r#String)
|   Output schema = __store_cluster_id_l#Utf8, __er_entity_pattern_id_l#Int64, __store_cluster_id_r#Utf8, __er_entity_pattern_id_r#Int64
|   Stats = { Approx num rows = 3,021,691, Approx size bytes = 81.41 MiB, Accumulated selectivity = 0.90 }
|\
| * Repartition: Scheme = Hash
| |   Num partitions = Some(200)
| |   By = col(__store_cluster_id_r)
| |   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }
| |
| * Project: col(__store_cluster_id) as __store_cluster_id_r, col(__er_entity_pattern_id) as __er_entity_pattern_id_r
| |   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }
| |
| * Num Scan Tasks = 16
| |   File schema = __store_cluster_id#Utf8, __er_records#List(Struct([Field { name: "__er_source_name", dtype: Utf8, metadata: {} }, Field { name: "__er_source_id", dtype: Utf8, metadata: {} }, Field { name: "__er_record_id", dtype: Utf8, metadata: {} }])), __er_entity_pattern_id#Int64
| |   Partitioning keys = []
| |   Projection pushdown = [__store_cluster_id, __er_entity_pattern_id]
| |   Filter pushdown = not(is_null(col(__store_cluster_id)))
| |   Output schema = __store_cluster_id#Utf8, __er_entity_pattern_id#Int64
| |   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }
|
* Repartition: Scheme = Hash
|   Num partitions = Some(200)
|   By = col(__store_cluster_id_l)
|   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }
|
* Project: col(__store_cluster_id) as __store_cluster_id_l, col(__er_entity_pattern_id) as __er_entity_pattern_id_l
|   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }
|
* Num Scan Tasks = 16
|   File schema = __store_cluster_id#Utf8, __er_records#List(Struct([Field { name: "__er_source_name", dtype: Utf8, metadata: {} }, Field { name: "__er_source_id", dtype: Utf8, metadata: {} }, Field { name: "__er_record_id", dtype: Utf8, metadata: {} }])), __er_entity_pattern_id#Int64
|   Partitioning keys = []
|   Projection pushdown = [__store_cluster_id, __er_entity_pattern_id]
|   Filter pushdown = not(is_null(col(__store_cluster_id)))
|   Output schema = __store_cluster_id#Utf8, __er_entity_pattern_id#Int64
|   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }


== Physical Plan ==

Repartition not supported on the NativeRunner. This will be a no-op. Please use the RayRunner instead if you need to repartition
Repartition not supported on the NativeRunner. This will be a no-op. Please use the RayRunner instead if you need to repartition
* Filter: col(4: __er_match_score) >= lit(1.0)
|   Stats = { Approx num rows = 271,953, Approx size bytes = 7.33 MiB, Accumulated selectivity = 0.08 }
|   Batch Size = Range(0, 131072]
|
* Project: col(0: __store_cluster_id_l), col(1: __er_entity_pattern_id_l), col(2: __store_cluster_id_r), col(3: __er_entity_pattern_id_r), cast(lit(1) as Float64) as __er_match_score
|   Stats = { Approx num rows = 906,508, Approx size bytes = 24.42 MiB, Accumulated selectivity = 0.27 }
|   Batch Size = Range(0, 131072]
|
* Filter: col(1: __er_entity_pattern_id_l) < col(3: __er_entity_pattern_id_r)
|   Stats = { Approx num rows = 906,508, Approx size bytes = 24.42 MiB, Accumulated selectivity = 0.27 }
|   Batch Size = Range(0, 131072]
|
* Inner Hash Join Probe:
|   Probe on: [col(0: __store_cluster_id_r)]
|   Build on left: true
|   Stats = { Approx num rows = 3,021,691, Approx size bytes = 81.41 MiB, Accumulated selectivity = 0.90 }
|   Batch Size = Range(0, 131072]
|\
| * Project: col(0: __store_cluster_id) as __store_cluster_id_r, col(1: __er_entity_pattern_id) as __er_entity_pattern_id_r
| |   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }
| |   Batch Size = Range(0, 131072]
| |
| * ScanTaskSource:
| |   Num Scan Tasks = 16
| |   Estimated Scan Bytes = 309469943
| |   Num Parallel Scan Tasks = 8
| |   Pushdowns: {projection: [__store_cluster_id, __er_entity_pattern_id], filter: not(is_null(col(__store_cluster_id)))}
| |   Schema: {__store_cluster_id#Utf8, __er_records#List(Struct([Field { name: "__er_source_name", dtype: Utf8, metadata: {} }, Field { name: "__er_source_id", dtype: Utf8, metadata: {} }, Field { name: "__er_record_id", dtype: Utf8, metadata: {} }])), __er_entity_pattern_id#Int64}
| |   Scan Tasks: [
| |   {File {file:///tmp/entity_patterns/ce9327e6-a33a-4d0a-a3cc-7f4abaaa8529-0.parquet}}
| |   {File {file:///tmp/entity_patterns/1f6b8db3-f268-4df8-a4dd-6461112b2310-0.parquet}}
| |   {File {file:///tmp/entity_patterns/13f8522e-cb85-4aeb-b8a8-e2669d98c734-0.parquet}}
| |   ...
| |   {File {file:///tmp/entity_patterns/bb9b516a-4c45-4765-aad3-fefee232ca5f-0.parquet}}
| |   {File {file:///tmp/entity_patterns/e653c3ff-65fa-4bd3-8e0b-5d3c0e9cb1b1-0.parquet}}
| |   {File {file:///tmp/entity_patterns/e6dc2a0a-cb99-4d90-8942-1eb1f9c40040-0.parquet}}
| |   ]
| |   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }
| |   Batch Size = Range(0, 131072]
|
* Hash Join Build:
|   Track Indices: true
|   Key Schema: __store_cluster_id_l#Utf8
|   Null equals Nulls = [false]
|   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }
|   Batch Size = Range(0, 131072]
|
* Project: col(0: __store_cluster_id) as __store_cluster_id_l, col(1: __er_entity_pattern_id) as __er_entity_pattern_id_l
|   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }
|   Batch Size = Range(0, 131072]
|
* ScanTaskSource:
|   Num Scan Tasks = 16
|   Estimated Scan Bytes = 309469943
|   Num Parallel Scan Tasks = 8
|   Pushdowns: {projection: [__store_cluster_id, __er_entity_pattern_id], filter: not(is_null(col(__store_cluster_id)))}
|   Schema: {__store_cluster_id#Utf8, __er_records#List(Struct([Field { name: "__er_source_name", dtype: Utf8, metadata: {} }, Field { name: "__er_source_id", dtype: Utf8, metadata: {} }, Field { name: "__er_record_id", dtype: Utf8, metadata: {} }])), __er_entity_pattern_id#Int64}
|   Scan Tasks: [
|   {File {file:///tmp/entity_patterns/ce9327e6-a33a-4d0a-a3cc-7f4abaaa8529-0.parquet}}
|   {File {file:///tmp/entity_patterns/1f6b8db3-f268-4df8-a4dd-6461112b2310-0.parquet}}
|   {File {file:///tmp/entity_patterns/13f8522e-cb85-4aeb-b8a8-e2669d98c734-0.parquet}}
|   ...
|   {File {file:///tmp/entity_patterns/bb9b516a-4c45-4765-aad3-fefee232ca5f-0.parquet}}
|   {File {file:///tmp/entity_patterns/e653c3ff-65fa-4bd3-8e0b-5d3c0e9cb1b1-0.parquet}}
|   {File {file:///tmp/entity_patterns/e6dc2a0a-cb99-4d90-8942-1eb1f9c40040-0.parquet}}
|   ]
|   Stats = { Approx num rows = 3,180,727, Approx size bytes = 85.69 MiB, Accumulated selectivity = 0.95 }
|   Batch Size = Range(0, 131072]

None