python_example/fss_dic_scraping_all.py at main · nicecoding1/python_example · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
import requests
from bs4 import BeautifulSoup
import re
import sqlite3

fdic = {}
list_a = []
list_b = []

for page in range(1, 55, 1):
    url = "https://fine.fss.or.kr/main/fin_tip/dic/financedic.jsp?page="+str(page)
    print(url)

    response = requests.get(url)

    if response.status_code == 200:
        html = response.text
        soup = BeautifulSoup(html, 'html.parser')
        # print(soup)
        ul = soup.select_one('ul.dic_result_list')
        sub = ul.select('li > dl > dt')
        con = ul.select('li > dl > dd')

        for i in sub:
            tmp = i.get_text().strip()
            tmp = tmp.replace("\r\n\t\t\t\t\t\t\t\t\xa0", "")
            tmp = re.sub('^[0-9]+. ', '', tmp)
            list_a.append(tmp)
        for i in con:
            tmp = i.get_text().strip()
            list_b.append(tmp)
    else:
        print(response.status_code)

for i in range(len(list_a)):
    fdic[i] = [list_a[i], list_b[i]]
# print(fdic)

conn = sqlite3.connect("fdic1.db")
cur = conn.cursor()
try:
    conn.execute("create table fss_dic(id integer, name text, content text)")
except Exception as e:
    print(e)

for i in fdic:
    name = fdic[i][0]
    content = fdic[i][1]

    sql = "insert into fss_dic values (?, ?, ?)"
    cur.execute(sql, (i, name, content))

conn.commit()
conn.close()