Spaces:
Sleeping
Sleeping
File size: 2,423 Bytes
77f290b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 |
import pandas as pd
import requests
import re
from multiprocessing import Pool, cpu_count
from functools import partial
# Function to process each URL
def process_paper(year, url):
try:
paper_page = requests.get(url).text
# Find title
title_pattern = r'<title>(.*?)\s*</title>'
title_match = re.search(title_pattern, paper_page, re.DOTALL)
title = title_match.group(1)
# Find the code repository link
code_repo_pattern = r'<h1 id="code-id">.*?</h1>\s*<p><a href="(.*?)">'
code_repo_match = re.search(code_repo_pattern, paper_page, re.DOTALL)
code_repo_link = code_repo_match.group(1) if code_repo_match else ""
# Find the dataset information
dataset_pattern = r'<h1 id="dataset-id">.*?</h1>\s*<p>(.*?)\s*<br />'
dataset_match = re.search(dataset_pattern, paper_page, re.DOTALL)
dataset_info = "Yes" if dataset_match else "No"
# Return a dictionary of the results
return {"title": title, "url": code_repo_link, "year": year, "public": dataset_info}
except Exception as e:
print(f"Error processing {url}: {e}")
return None
current_year = 2024 # Update with the current year
MICCAI_pages = ["https://miccai2021.org/openaccess/paperlinks/", "https://conferences.miccai.org/2022/papers/", "https://conferences.miccai.org/2023/papers/"]
MICCAI_root = ["https://miccai2021.org/openaccess/paperlinks/", "https://conferences.miccai.org", "https://conferences.miccai.org"]
years = [2021, 2022, 2023]
# Set debug mode
debug = False
# Fetch all URLs for each year
all_year_urls = []
for i in range(len(MICCAI_pages)):
year_page = requests.get(MICCAI_pages[i]).text
print(year_page)
urls = [MICCAI_root[i] + line.split('href="')[1].split('"')[0] for line in year_page.split('\n') if "&bullet" in line]
all_year_urls.extend([(years[i], url) for url in urls])
print(all_year_urls)
# Parallel processing using Pool
# if __name__ == "__main__":
# with Pool(processes=12) as pool: # Use 12 processes
# results = pool.starmap(process_paper, all_year_urls)
# # Filter out any None results due to errors
# results = [result for result in results if result is not None]
# miccai = pd.DataFrame(results)
# # miccai = pd.DataFrame( OrderedDict( { 'title': pd.Series(a), 'b': pd.Series(b), 'c': pd.Series(c) } ) )
# miccai.to_csv('miccai.csv') |