Skip to content

Commit

Permalink
faet(#168): filtered.txt
Browse files Browse the repository at this point in the history
  • Loading branch information
h1alexbel committed Dec 31, 2024
1 parent 6bb267b commit 27f898c
Show file tree
Hide file tree
Showing 5 changed files with 9 additions and 4 deletions.
1 change: 1 addition & 0 deletions data.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ done
cp "collect.log" "collection/$OUT"
cp "numerical.csv" "collection/$OUT/d0-numerical.csv"
cp "scores.csv" "collection/$OUT/d1-scores.csv"
cp "filtered.txt" "collection/$OUT"
if "$EMBEDDINGS"; then
cp "sbert.csv" "collection/$OUT/d2-sbert.csv"
cp "e5.csv" "collection/$OUT/d3-e5.csv"
Expand Down
4 changes: 2 additions & 2 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ license_filter repos out="experiment/after-license-filter.csv":
cd sr-data && poetry poe license_filter --repos "{{repos}}" --out "{{out}}"

# Filter collected repositories.
filter repos removed out="experiment/after-filter.csv":
cd sr-data && poetry poe filter --repos "{{repos}}" --out "{{out}}" --removed "{{removed}}"
filter repos filtered out="experiment/after-filter.csv":
cd sr-data && poetry poe filter --repos "{{repos}}" --out "{{out}}" --filtered "{{filtered}}"

# Extract headings from README files.
extract repos out="experiment/after-extract.csv":
Expand Down
4 changes: 2 additions & 2 deletions sr-data/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ args = [
]

[tool.poe.tasks.filter]
script = "sr_data.steps.filter:main(repos, out, removed)"
args = [{name = "repos"}, {name = "out"}, {name = "removed"}]
script = "sr_data.steps.filter:main(repos, out, filtered)"
args = [{name = "repos"}, {name = "out"}, {name = "filtered"}]

[tool.poe.tasks.license_filter]
script = "sr_data.steps.license_filter:main(repos, out)"
Expand Down
1 change: 1 addition & 0 deletions sr-data/resources/pipeline.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
},
"filter": {
"repos": "@in",
"filtered": "../filtered.txt",
"out": "../after-filter.csv"
},
"extract": {
Expand Down
3 changes: 3 additions & 0 deletions sr-data/src/sr_data/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ def main(representation, steps, pipes, out):
if "token" in params:
token = params["token"]
command += f" {token}"
if "filtered" in params:
filtered = params["filtered"]
command += f" \"{filtered}\""
if "out" in params:
output = params["out"]
command += f" \"{output}\""
Expand Down

0 comments on commit 27f898c

Please sign in to comment.