Python爬虫---中国大学MOOC爬取数据（文中有数据集）

码农世界 2024-05-22 后端 78 次浏览 0个评论

1、内容简介

本文为大二在校学生所做，内容为爬取中国大学Mooc网站的课程分类数据、课程数据、评论数据。数据集大佬们需要拿走。主要是希望大佬们能指正代码问题。

2、数据集

课程评论数据集，343525条（包括评论id、评论时间、发送评论用户的id，发送评论的用户昵称，评论内容，所属课程id）

课程数据集，29196条（包括课程id、课程、课程名称、报名人数、教师名称、所属大学、开始日期、截止日期、所属课程类别id）

课程分类数据集，23条（包括课程id、课程名称、课程简介）

链接：https://pan.baidu.com/s/10m_kWEvLaom41sFvb8CQgA?pwd=8888

提取码：8888

--来自百度网盘超级会员V4的分享

3、代码

3.1 获取课程类别

3.1.1 全部代码

import csv
import json
import requests
# 获取类别的数据
# 请求url
baseurl = "https://www.icourse163.org/web/j/channelBean.listChannelCategoryDetail.rpc?csrfKey=9ddd9641afce4905aa429bf754db5b1b"
# 请求头
headers = {
"Accept":"*/*",
"Accept-Encoding":"gzip, deflate, br",
"Accept-Language":"zh-CN,zh;q=0.9",
"Cache-Control":"no-cache",
"Content-Length":"192",
"Content-Type":"application/x-www-form-urlencoded;charset=UTF-8",
"Cookie":"NTESSTUDYSI=9ddd9641afce4905aa429bf754db5b1b; EDUWEBDEVICE=7a18a0d811e443da8466a956d9571abd; Hm_lvt_77dc9a9d49448cf5e629e5bebaa5500b=1714007316; __yadk_uid=lxVu0GBpcgHrjnjdBP4LXojZlFfwpS4n; WM_NI=4G7PWoEQ02m4%2BvueYIsLLEdVSrkmtq2QbnYz6oN0vU7DuVNiljn7xkLMqPibUCA0Y3KTw4e3PffcgFfwwieW1RRmO7vvCHyP7%2FfjlmJia7I03OR%2FMP0xMocc%2FWmx3lHnckU%3D; WM_NIKE=9ca17ae2e6ffcda170e2e6ee91e159a5adb6b5f162b1a88aa6d15f968b9e86d83eb0bc83dae67dbc8699a6db2af0fea7c3b92af899f9b1d665878f00d3f040e9aabb86dc43b19484a6e75e91aea58ef77bf28fae8fdc72f6b182d1f55ff59cbbb7cd63b0b6a2d8fb619093bfb1ec6a979d8fd1dc39898cb9d5e83994baaa84e64ea6a9b98ae85df38e9e8ece3db6aa8da9fc5cf5f1aa8bef678ab7afd8cc3cf1919ed0d05cf4a88390e269ed89a399b2688cb2ad8cd837e2a3; WM_TID=uZLSXz1D89ZBAUAUVQKFu0YZ91x1YWRD; Hm_lpvt_77dc9a9d49448cf5e629e5bebaa5500b=1714009886",
"Edu-Script-Token":"9ddd9641afce4905aa429bf754db5b1b",
"Origin":"https://www.icourse163.org",
"Pragma":"no-cache",
"Referer":"https://www.icourse163.org/channel/2001.htm?cate=-1&subCate=-1",
"Sec-Ch-Ua":'"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
"Sec-Ch-Ua-Mobile":"?1",
"Sec-Ch-Ua-Platform":"Android",
"Sec-Fetch-Dest":"empty",
"Sec-Fetch-Mode":"cors",
"Sec-Fetch-Site":"same-origin",
"User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.289 Mobile Safari/537.36"
}
# 请求数据，固定的
data = "includeALLChannels=true&includeDefaultChannels=false&includeMyChannels=false";
# 发送请求
response = requests.post(baseurl,data=data, headers=headers,timeout=10)
# 获取数据
parsed_data = json.loads(response.text)
# 将数据转为utf-8模式
for key, value in parsed_data.items():
    if isinstance(value, str):
        parsed_data[key] = value.encode('utf-8').decode('unicode-escape')
# 打印 类别JSON数据
print(json.dumps(parsed_data, indent=4, ensure_ascii=False))
# 定义需要写入csv的字段
fields = ['id', 'name', 'shortDesc', 'weight', 'defaultChannel', 'charge', 'includeLearningProgress', 'newGiftBag', 'suportOoc', 'suportVocationalMooc', 'suportNavigationType', 'icon', 'recommendWord', 'seoKeywords', 'showChildCategory', 'childrenChannelIds', 'childrenChannelDtoList']
# 创建 CSV 文件并写入数据
with open('channel_data.csv', 'w', newline='', encoding='utf-8-sig') as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(fields)
    # 按照json数据 遍历写入数据即可
    for channel_category in parsed_data['result']['channelCategoryDetails']:
        for channel in channel_category['channels']:
            row = [channel[field] for field in fields]
            writer.writerow(row)

3.1.2 代码解释

中国大学Mooc的网站爬取很简单，几乎没有反爬手段，适合我这种初学者。

首先，是请求头数据的获取。浏览器右键打开审查模式，单击网络，找到一个请求，将其请求头数据复制下来。

请求的url我就不多说了，这个是我自己慢慢看后搜索出来的

我们可以看到，该请求方法为post，载荷，也就是携带的参数是固定的。那么简单，我们就把参数转成字符串，封装为data，一起给请求头发过去。

之后我们获取数据，并转为json格式。再把json数据转为utf-8的编码格式，防止乱码。这些数据处理就不细讲了。

然后我们根据他的json格式，遍历数据，最后输出为csv文件。

3.2 获取全部课程

3.2.1 全部代码

import csv
import json
import requests
# 目的：获取每个课程的概览信息
# 先创建一个csv文件，把列名写进去
with open('course_data.csv', 'w', newline='', encoding='utf-8-sig') as csvfile:
    writer = csv.writer(csvfile)
    # 设置课程数据的列名
    fields = ['id', 'name', 'enrollCount', 'teacherName', 'schoolName', 'startTime', 'endTime']
    # 设置所有数据的列名（因为有些数据不在课程信息的VO类里，需要单独获取，所以另外设立）
    all = ['id', 'name', 'enrollCount', 'teacherName', 'schoolName', 'startTime', 'endTime','type',"channel_Id"]
    writer.writerow(all)
# 定义获取课程类别的url
baseurl = "https://www.icourse163.org/web/j/channelBean.listChannelCategoryDetail.rpc?csrfKey=9ddd9641afce4905aa429bf754db5b1b"
# 定义获取课程信息的url
twourl = "https://www.icourse163.org/web/j/mocSearchBean.searchCourseCardByChannelAndCategoryId.rpc?csrfKey=9ddd9641afce4905aa429bf754db5b1b"
# 请求头
headers = {
"Accept":"*/*",
"Accept-Encoding":"gzip, deflate, br",
"Accept-Language":"zh-CN,zh;q=0.9",
"Cache-Control":"no-cache",
"Content-Length":"192",
"Content-Type":"application/x-www-form-urlencoded;charset=UTF-8",
"Cookie":"NTESSTUDYSI=9ddd9641afce4905aa429bf754db5b1b; EDUWEBDEVICE=7a18a0d811e443da8466a956d9571abd; Hm_lvt_77dc9a9d49448cf5e629e5bebaa5500b=1714007316; __yadk_uid=lxVu0GBpcgHrjnjdBP4LXojZlFfwpS4n; WM_NI=4G7PWoEQ02m4%2BvueYIsLLEdVSrkmtq2QbnYz6oN0vU7DuVNiljn7xkLMqPibUCA0Y3KTw4e3PffcgFfwwieW1RRmO7vvCHyP7%2FfjlmJia7I03OR%2FMP0xMocc%2FWmx3lHnckU%3D; WM_NIKE=9ca17ae2e6ffcda170e2e6ee91e159a5adb6b5f162b1a88aa6d15f968b9e86d83eb0bc83dae67dbc8699a6db2af0fea7c3b92af899f9b1d665878f00d3f040e9aabb86dc43b19484a6e75e91aea58ef77bf28fae8fdc72f6b182d1f55ff59cbbb7cd63b0b6a2d8fb619093bfb1ec6a979d8fd1dc39898cb9d5e83994baaa84e64ea6a9b98ae85df38e9e8ece3db6aa8da9fc5cf5f1aa8bef678ab7afd8cc3cf1919ed0d05cf4a88390e269ed89a399b2688cb2ad8cd837e2a3; WM_TID=uZLSXz1D89ZBAUAUVQKFu0YZ91x1YWRD; Hm_lpvt_77dc9a9d49448cf5e629e5bebaa5500b=1714009886",
"Edu-Script-Token":"9ddd9641afce4905aa429bf754db5b1b",
"Origin":"https://www.icourse163.org",
"Pragma":"no-cache",
"Referer":"https://www.icourse163.org/channel/2001.htm?cate=-1&subCate=-1",
"Sec-Ch-Ua":'"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
"Sec-Ch-Ua-Mobile":"?1",
"Sec-Ch-Ua-Platform":"Android",
"Sec-Fetch-Dest":"empty",
"Sec-Fetch-Mode":"cors",
"Sec-Fetch-Site":"same-origin",
"User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.289 Mobile Safari/537.36"
}
# 我们先进行获取所有类别的请求
data = "includeALLChannels=true&includeDefaultChannels=false&includeMyChannels=false";
response = requests.post(baseurl,data=data, headers=headers,timeout=10)
# 接收所有类别的数据
parsed_data = json.loads(response.text)
# 转换数据为utf-8编码格式
for key, value in parsed_data.items():
    if isinstance(value, str):
        parsed_data[key] = value.encode('utf-8').decode('unicode-escape')
    # 遍历所有类别
    for channel_category in parsed_data['result']['channelCategoryDetails']:
        for channel in channel_category['channels']:
            # 当前页码
            pageIndex = 1
            # 我也不知道这个参数干啥的，根据正常请求里的数据 无脑-1就完了
            categoryId = -1;
            # 获取类别的id
            categoryChannelId = channel["id"];
            # 进行循环，要进行 获取类别内所有课程信息 的请求
            while True :
                # 设置请求参数
                data = {
                    "mocCourseQueryVo": '{categoryId:' + str(categoryId) + ',categoryChannelId:' + str(
                        categoryChannelId) + ','
                                             'orderBy:0,stats:30,pageIndex:' + str(
                        pageIndex) + ',pageSize:20,shouldConcatData:true}'
                }
                try:
                    # 发送请求，因为请求头相同，不做赘述
                    response = requests.post(twourl, data=data, headers=headers)
                    # 当前页数+1，完成下次翻页操作
                    pageIndex+=1
                    # 获取该类别内课程的数据
                    parsed_course_data = json.loads(response.text)
                    # 转化为utf-8编码格式
                    for key, value in parsed_data.items():
                        if isinstance(value, str):
                            parsed_course_data[key] = value.encode('utf-8').decode('unicode-escape')
                    # 打印获取的 课程JSON数据
                    print(json.dumps(parsed_course_data, indent=4, ensure_ascii=False))
                    # 创建 CSV 文件并写入数据
                    with open('course_data.csv', 'a', newline='', encoding='utf-8-sig') as csvfile:
                        writer = csv.writer(csvfile)
                        # 遍历类别的课程数据
                        for item in parsed_course_data['result']['list']:
                            # 获取课程数据
                            course = item.get('mocCourseBaseCardVo')
                            # 获取type数据（我也不知道干啥的）
                            type = item.get('type')
                            # 获取该课程所属类别的id
                            channelId = categoryChannelId
                            # 在csv写入数据
                            if course:
                                row = [course.get(field, '') for field in fields]
                                if type:
                                    row.append(type)
                                if channelId:
                                    row.append(channelId)
                                writer.writerow(row)
                        # 如果翻页数超过或等于最大页码，退出循环
                        if pageIndex >= parsed_course_data["result"]["query"]["totlePageCount"]:
                            break;
                except Exception as e:
                    print(f"网络异常,{e}")
                    continue

3.2.2 代码解释

课程获取的请求URL如下图所示，我找的这个URL的过程就是，先筛选为Fetch/XHR，在点击每个链接，看他的响应。找到之后我们注意他的请求方式和url即可。

之后我们看他的载荷，也就是请求数据。我们可以看到，是一个请求头。包括类别id，页码，页数等，一个很常规的查询请求参数。我们只需把请求体封装为字符形式即可。

我们在3.1 获取课程类别的基础上，遍历课程类别数据，之后通过类别id，搜索对应课程数据。之后根据课程数据的json格式，依次遍历数据即可，除此之外，我们还可以额外加一个课程类别id的列，便于更好管理。

3.3 获取评论

3.3.1 全部代码

# 获取课程大纲
# dagangurl = 'https://www.icourse163.org/web/j/courseBean.getMocTermDto.rpc?csrfKey=9ddd9641afce4905aa429bf754db5b1b'
import csv
import json
import requests
# 目的：获取每个课程的评论
# 先创建一个csv文件，把列名写进去
with open('course_comment_data.csv', 'w', newline='', encoding='utf-8-sig') as csvfile:
    writer = csv.writer(csvfile)
    # 设置课程数据的列名
    fields = ['id',"gmtModified","commentorId", 'userNickName', 'content', 'mark', 'schoolName', 'termId', 'status']
    # 设置所有数据的列名（因为有些数据不在课程信息的VO类里，需要单独获取，所以另外设立）
    all = ['id',"gmtModified","commentorId", 'userNickName', 'content', 'mark', 'schoolName', 'termId', 'status','courseId']
    writer.writerow(all)
# 定义获取课程类别的url
baseurl = "https://www.icourse163.org/web/j/channelBean.listChannelCategoryDetail.rpc?csrfKey=9ddd9641afce4905aa429bf754db5b1b"
# 定义获取课程信息的url
twourl = "https://www.icourse163.org/web/j/mocSearchBean.searchCourseCardByChannelAndCategoryId.rpc?csrfKey=9ddd9641afce4905aa429bf754db5b1b"
# 获取课程评论信息的url
commentUrl = 'https://www.icourse163.org/web/j/mocCourseV2RpcBean.getCourseEvaluatePaginationByCourseIdOrTermId.rpc?csrfKey=9ddd9641afce4905aa429bf754db5b1b'
# 请求头
headers = {
    "Accept":"*/*",
    "Accept-Encoding":"gzip, deflate, br",
    "Accept-Language":"zh-CN,zh;q=0.9",
    "Cache-Control":"no-cache",
    "Content-Length":"192",
    "Content-Type":"application/x-www-form-urlencoded;charset=UTF-8",
    "Cookie":"NTESSTUDYSI=9ddd9641afce4905aa429bf754db5b1b; EDUWEBDEVICE=7a18a0d811e443da8466a956d9571abd; Hm_lvt_77dc9a9d49448cf5e629e5bebaa5500b=1714007316; __yadk_uid=lxVu0GBpcgHrjnjdBP4LXojZlFfwpS4n; WM_NI=4G7PWoEQ02m4%2BvueYIsLLEdVSrkmtq2QbnYz6oN0vU7DuVNiljn7xkLMqPibUCA0Y3KTw4e3PffcgFfwwieW1RRmO7vvCHyP7%2FfjlmJia7I03OR%2FMP0xMocc%2FWmx3lHnckU%3D; WM_NIKE=9ca17ae2e6ffcda170e2e6ee91e159a5adb6b5f162b1a88aa6d15f968b9e86d83eb0bc83dae67dbc8699a6db2af0fea7c3b92af899f9b1d665878f00d3f040e9aabb86dc43b19484a6e75e91aea58ef77bf28fae8fdc72f6b182d1f55ff59cbbb7cd63b0b6a2d8fb619093bfb1ec6a979d8fd1dc39898cb9d5e83994baaa84e64ea6a9b98ae85df38e9e8ece3db6aa8da9fc5cf5f1aa8bef678ab7afd8cc3cf1919ed0d05cf4a88390e269ed89a399b2688cb2ad8cd837e2a3; WM_TID=uZLSXz1D89ZBAUAUVQKFu0YZ91x1YWRD; Hm_lpvt_77dc9a9d49448cf5e629e5bebaa5500b=1714009886",
    "Edu-Script-Token":"9ddd9641afce4905aa429bf754db5b1b",
    "Origin":"https://www.icourse163.org",
    "Pragma":"no-cache",
    "Referer":"https://www.icourse163.org/channel/2001.htm?cate=-1&subCate=-1",
    "Sec-Ch-Ua":'"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
    "Sec-Ch-Ua-Mobile":"?1",
    "Sec-Ch-Ua-Platform":"Android",
    "Sec-Fetch-Dest":"empty",
    "Sec-Fetch-Mode":"cors",
    "Sec-Fetch-Site":"same-origin",
    "User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.289 Mobile Safari/537.36"
}
# 我们先进行获取所有类别的请求
# 请求数据，固定的
data = "includeALLChannels=true&includeDefaultChannels=false&includeMyChannels=false";
# 发送请求
response = requests.post(baseurl,data=data, headers=headers,timeout=10)
# 接收所有类别的数据
parsed_data = json.loads(response.text)
# 转换数据为utf-8编码格式
for key, value in parsed_data.items():
    if isinstance(value, str):
        parsed_data[key] = value.encode('utf-8').decode('unicode-escape')
    # 遍历所有类别
    for channel_category in parsed_data['result']['channelCategoryDetails']:
        for channel in channel_category['channels']:
            # 当前页码
            pageIndex = 1
            # 我也不知道这个参数干啥的，根据正常请求里的数据 无脑-1就完了
            categoryId = -1;
            # 获取类别的id
            categoryChannelId = channel["id"];
            # 进行循环，要进行 获取类别内所有课程信息 的请求
            while True :
                # 设置请求参数
                data = {
                    "mocCourseQueryVo": '{categoryId:' + str(categoryId) + ',categoryChannelId:' + str(
                        categoryChannelId) + ','
                                             'orderBy:0,stats:30,pageIndex:' + str(
                        pageIndex) + ',pageSize:20,shouldConcatData:true}'
                }
                try:
                    # 发送请求，因为请求头相同，不做赘述
                    response = requests.post(twourl, data=data, headers=headers)
                    # 当前页数+1，完成下次翻页操作
                    pageIndex+=1
                    # 获取该类别内课程的数据
                    parsed_course_data = json.loads(response.text)
                    # 转化为utf-8编码格式
                    for key, value in parsed_data.items():
                        if isinstance(value, str):
                            parsed_course_data[key] = value.encode('utf-8').decode('unicode-escape')
                     # 遍历类别的课程数据
                    for item in parsed_course_data['result']['list']:
                            # 设置获取评论请求的数据（获取课程id、当前页码）
                            courseId = item.get('mocCourseBaseCardVo').get("id")
                            commentPageIndex = 1;
                            # 根据id获取对应课程评论
                            while True :
                                commentData = {
                                    'courseId': str(courseId),
                                    'pageIndex': str(commentPageIndex),
                                    'pageSize': str(20),
                                    'orderBy': str(3)
                                }
                                response = requests.post(commentUrl, data=commentData, headers=headers, timeout=10)
                                commentResultData = json.loads(response.text)
                                commentPageIndex += 1
                                # 打印获取的 课程JSON数据
                                print(json.dumps(commentResultData, indent=4, ensure_ascii=False))
                                # 创建 CSV 文件并写入数据
                                with open('course_comment_data.csv', 'a', newline='', encoding='utf-8-sig') as csvfile:
                                    writer = csv.writer(csvfile)
                                    # 遍历类别的课程数据
                                    for item in commentResultData['result']['list']:
                                        # 在csv写入数据
                                        if item:
                                            row = [item.get(field, '') for field in fields]
                                            row.append(courseId)
                                            writer.writerow(row)
                                # 如果 课程评论数据 翻页数超过或等于最大页码，退出循环
                                if commentPageIndex >= commentResultData["result"]["query"]["totlePageCount"]:
                                    break;
                    # 如果课程数据翻页数超过或等于最大页码，退出循环
                    if pageIndex >= parsed_course_data["result"]["query"]["totlePageCount"]:
                        break;
                except Exception as e:
                    print(f"网络异常,{e}")
                    break;

转载请注明来自码农世界，本文标题：《Python爬虫---中国大学MOOC爬取数据（文中有数据集）》