Spaces:
Running
Running
File size: 2,095 Bytes
82334b0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
# Copyright 2024 LY Corporation
# LY Corporation licenses this file to you under the Apache License,
# version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at:
# https://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from pathlib import Path
import hydra
import pandas as pd
from omegaconf import DictConfig
def split(df):
unique_spk_ids = df["spk_id"].unique()
split_idx = int(len(unique_spk_ids) * 0.98)
trn_spk_ids = unique_spk_ids[:split_idx]
val_spk_ids = unique_spk_ids[split_idx:]
trn_df = df[df["spk_id"].isin(trn_spk_ids)]
val_df = df[df["spk_id"].isin(val_spk_ids)]
trn_df = trn_df.sort_values(by=["item_name"])
val_df = val_df.sort_values(by=["item_name"])
return trn_df, val_df
@hydra.main(version_base=None, config_path="conf/", config_name="preprocess")
def main(cfg: DictConfig):
df_dir = Path(cfg.path.df_dir)
filtered_df_dir = Path(cfg.path.filtered_df_dir)
filtered_df_dir.mkdir(exist_ok=True)
df = pd.read_csv(df_dir / "train.csv")
data_df = pd.read_csv(cfg.path.data_csv_file)
data_df = data_df[data_df["invalid"] == 0]
print(df.shape, data_df.shape)
df = df[df["item_name"].isin(data_df["item_name"])]
print(df.shape)
merged_df = pd.merge(
df, data_df[["item_name", "style_prompt_key"]], on="item_name", how="left"
)
merged_df = merged_df.drop(columns="style_prompt_key_x")
df = merged_df.rename(columns={"style_prompt_key_y": "style_prompt_key"})
trn_df, val_df = split(df)
trn_df.to_csv(filtered_df_dir / "trn.csv", index=False)
val_df.to_csv(filtered_df_dir / "val.csv", index=False)
print(trn_df.shape, val_df.shape)
if __name__ == "__main__":
main()
|