Upload folder using huggingface_hub

.gitattributes

@@ -79,3 +79,18 @@ data/datasets/lilac/imdb/text/text_statistics/data-00000-of-00001.parquet filter
 data/datasets/lilac/databricks-dolly-15k-curated-en/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/databricks-dolly-15k-curated-en/new-context/value/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
 data/datasets/lilac/databricks-dolly-15k-curated-en/original-context/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/question/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/question/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/question/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/question/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/question/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/question/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/question/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/response/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/response/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/response/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/response/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/response/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/response/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
+data/datasets/lilac/OpenOrca-100k/response/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text

data/datasets/lilac/OpenOrca-100k/config.yml

@@ -0,0 +1,28 @@
+embeddings:
+- {embedding: gte-small, path: response}
+- {embedding: gte-small, path: question}
+name: OpenOrca-100k
+namespace: local
+settings:
+  preferred_embedding: gte-small
+  ui:
+    media_paths: [question, response]
+signals:
+- path: question
+  signal: {signal_name: near_dup}
+- path: question
+  signal: {signal_name: text_statistics}
+- path: question
+  signal: {signal_name: pii}
+- path: question
+  signal: {signal_name: lang_detection}
+- path: response
+  signal: {signal_name: near_dup}
+- path: response
+  signal: {signal_name: text_statistics}
+- path: response
+  signal: {signal_name: pii}
+- path: response
+  signal: {signal_name: lang_detection}
+source: {dataset_name: Open-Orca/OpenOrca, sample_size: 100000, source_name: huggingface}
+tags: [machine-learning]

data/datasets/lilac/OpenOrca-100k/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f95588367446af55ccc2f089092779670c57308ee1f72a849e41f22e126d5052
+size 105147761

data/datasets/lilac/OpenOrca-100k/manifest.json

@@ -0,0 +1,24 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "data_schema": {
+    "fields": {
+      "id": {
+        "dtype": "string"
+      },
+      "system_prompt": {
+        "dtype": "string"
+      },
+      "question": {
+        "dtype": "string"
+      },
+      "response": {
+        "dtype": "string"
+      },
+      "__hfsplit__": {
+        "dtype": "string"
+      }
+    }
+  }
+}

data/datasets/lilac/OpenOrca-100k/question/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f154c2dc5a0d69538c39df10508fe05cc36fb5489b61c303c9869320ef04581
+size 596704812

data/datasets/lilac/OpenOrca-100k/question/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c0bac1790aa5247eb288c2a828a92eb313090b36a015665f6aae42e5a4dcb18
+size 9378299

data/datasets/lilac/OpenOrca-100k/question/gte-small/signal_manifest.json

@@ -0,0 +1,32 @@
+{
+  "files": [],
+  "parquet_id": "question.gte-small",
+  "data_schema": {
+    "fields": {
+      "question": {
+        "fields": {
+          "gte-small": {
+            "repeated_field": {
+              "fields": {
+                "embedding": {
+                  "dtype": "embedding"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "signal_name": "gte-small"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "question"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/OpenOrca-100k/question/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef19c506d4af4eab34aec3b280663687002db0792108b84d313f8ab6f532aa6c
+size 6922769

data/datasets/lilac/OpenOrca-100k/question/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1a09e7085a4019205c62c28e6dcb46254fea37243e8087346d9c7298e05f9e1
+size 3327888

data/datasets/lilac/OpenOrca-100k/question/lang_detection/signal_manifest.json

@@ -0,0 +1,28 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "question.lang_detection",
+  "data_schema": {
+    "fields": {
+      "question": {
+        "fields": {
+          "lang_detection": {
+            "dtype": "string",
+            "signal": {
+              "split_by_paragraph": false,
+              "signal_name": "lang_detection"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "question"
+  ]
+}

data/datasets/lilac/OpenOrca-100k/question/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1ff51f57fb136ab846d0c34a248aca4ef86d09fa0945737cd2c276d2f5dcb7d
+size 3884385

data/datasets/lilac/OpenOrca-100k/question/near_dup/signal_manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "question.near_dup",
+  "data_schema": {
+    "fields": {
+      "question": {
+        "fields": {
+          "near_dup": {
+            "fields": {
+              "cluster_id": {
+                "dtype": "uint32",
+                "categorical": true
+              }
+            },
+            "signal": {
+              "threshold": 0.85,
+              "signal_name": "near_dup"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "question"
+  ]
+}

data/datasets/lilac/OpenOrca-100k/question/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acc4cd2ae7c51b4450d159c63fee3e9739b3c1d5a36cfbf3bf45fe29e2ac15b5
+size 3317869

data/datasets/lilac/OpenOrca-100k/question/pii/signal_manifest.json

@@ -0,0 +1,42 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "question.pii",
+  "data_schema": {
+    "fields": {
+      "question": {
+        "fields": {
+          "pii": {
+            "fields": {
+              "emails": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "ip_addresses": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "secrets": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              }
+            },
+            "signal": {
+              "signal_name": "pii"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "question"
+  ]
+}

data/datasets/lilac/OpenOrca-100k/question/text_statistics/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6703f93314760ee15d64532812a601c85d2f411254c1d809c6b3f558cc1c7c7
+size 4321496

data/datasets/lilac/OpenOrca-100k/question/text_statistics/signal_manifest.json

@@ -0,0 +1,56 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "question.text_statistics",
+  "data_schema": {
+    "fields": {
+      "question": {
+        "fields": {
+          "text_statistics": {
+            "fields": {
+              "num_characters": {
+                "dtype": "int32"
+              },
+              "readability": {
+                "dtype": "float32"
+              },
+              "log(type_token_ratio)": {
+                "dtype": "float32"
+              },
+              "frac_non_ascii": {
+                "dtype": "float32",
+                "bins": [
+                  [
+                    "Low",
+                    null,
+                    0.15
+                  ],
+                  [
+                    "Medium",
+                    0.15,
+                    0.3
+                  ],
+                  [
+                    "High",
+                    0.3,
+                    null
+                  ]
+                ]
+              }
+            },
+            "signal": {
+              "signal_name": "text_statistics"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "question"
+  ]
+}

data/datasets/lilac/OpenOrca-100k/response/gte-small/hnsw.hnswlib.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2374770842450f7d1712e2d56bc2e50bb1579af4cda061df2baf4631965dbcd
+size 482647596

data/datasets/lilac/OpenOrca-100k/response/gte-small/hnsw.lookup.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afd6636965df6ed8f6aadd52a9638edf201c36dd470b816e6488e5417dcfe3c4
+size 8159214

data/datasets/lilac/OpenOrca-100k/response/gte-small/signal_manifest.json

@@ -0,0 +1,32 @@
+{
+  "files": [],
+  "parquet_id": "response.gte-small",
+  "data_schema": {
+    "fields": {
+      "response": {
+        "fields": {
+          "gte-small": {
+            "repeated_field": {
+              "fields": {
+                "embedding": {
+                  "dtype": "embedding"
+                }
+              },
+              "dtype": "string_span"
+            },
+            "signal": {
+              "signal_name": "gte-small"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "gte-small"
+  },
+  "enriched_path": [
+    "response"
+  ],
+  "vector_store": "hnsw"
+}

data/datasets/lilac/OpenOrca-100k/response/gte-small/spans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf61917d291a1e3157ca017b4eacdf8983bf8094b3b22d710031381927f19b16
+size 6373377

data/datasets/lilac/OpenOrca-100k/response/lang_detection/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62af2b56e9bf3cbcddbceab6f858fc35fef50953b73b06a7da3bc1d2e62d3a53
+size 3339983

data/datasets/lilac/OpenOrca-100k/response/lang_detection/signal_manifest.json

@@ -0,0 +1,28 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "response.lang_detection",
+  "data_schema": {
+    "fields": {
+      "response": {
+        "fields": {
+          "lang_detection": {
+            "dtype": "string",
+            "signal": {
+              "split_by_paragraph": false,
+              "signal_name": "lang_detection"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "split_by_paragraph": false,
+    "signal_name": "lang_detection"
+  },
+  "enriched_path": [
+    "response"
+  ]
+}

data/datasets/lilac/OpenOrca-100k/response/near_dup/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc30679d1a2e6f2d3d45b2f145932daebf8a3f6ae4b73cfa9da3dbf5c495967d
+size 3902985

data/datasets/lilac/OpenOrca-100k/response/near_dup/signal_manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "response.near_dup",
+  "data_schema": {
+    "fields": {
+      "response": {
+        "fields": {
+          "near_dup": {
+            "fields": {
+              "cluster_id": {
+                "dtype": "uint32",
+                "categorical": true
+              }
+            },
+            "signal": {
+              "threshold": 0.85,
+              "signal_name": "near_dup"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "threshold": 0.85,
+    "signal_name": "near_dup"
+  },
+  "enriched_path": [
+    "response"
+  ]
+}

data/datasets/lilac/OpenOrca-100k/response/pii/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3b5830b33a8ddbe000b1b4403ef882731243075acc6416b5f673c90d4bf25ac
+size 3313965

data/datasets/lilac/OpenOrca-100k/response/pii/signal_manifest.json

@@ -0,0 +1,42 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "response.pii",
+  "data_schema": {
+    "fields": {
+      "response": {
+        "fields": {
+          "pii": {
+            "fields": {
+              "emails": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "ip_addresses": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              },
+              "secrets": {
+                "repeated_field": {
+                  "dtype": "string_span"
+                }
+              }
+            },
+            "signal": {
+              "signal_name": "pii"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "pii"
+  },
+  "enriched_path": [
+    "response"
+  ]
+}

data/datasets/lilac/OpenOrca-100k/response/text_statistics/data-00000-of-00001.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e5bae031d37b7df9a3df49a616d58a8f9962307750039c1736b8faa56d8501a
+size 4281305

data/datasets/lilac/OpenOrca-100k/response/text_statistics/signal_manifest.json

@@ -0,0 +1,56 @@
+{
+  "files": [
+    "data-00000-of-00001.parquet"
+  ],
+  "parquet_id": "response.text_statistics",
+  "data_schema": {
+    "fields": {
+      "response": {
+        "fields": {
+          "text_statistics": {
+            "fields": {
+              "num_characters": {
+                "dtype": "int32"
+              },
+              "readability": {
+                "dtype": "float32"
+              },
+              "log(type_token_ratio)": {
+                "dtype": "float32"
+              },
+              "frac_non_ascii": {
+                "dtype": "float32",
+                "bins": [
+                  [
+                    "Low",
+                    null,
+                    0.15
+                  ],
+                  [
+                    "Medium",
+                    0.15,
+                    0.3
+                  ],
+                  [
+                    "High",
+                    0.3,
+                    null
+                  ]
+                ]
+              }
+            },
+            "signal": {
+              "signal_name": "text_statistics"
+            }
+          }
+        }
+      }
+    }
+  },
+  "signal": {
+    "signal_name": "text_statistics"
+  },
+  "enriched_path": [
+    "response"
+  ]
+}