jd_data_proc_api.py

jd_data_proc_api.py — 京东招聘直连

POST 接口拿职位 JSON，generate_job_html 把结构化数据拼成完整 HTML 落盘，无缝进入后续 ann_md → ann_model 解析流水线。

代码

import time
import hashlib
import os
import requests
from urllib.parse import urlencode
import sys

sys.path.append('../')
import json
from utils import ner_logger
import re

# 请求头配置，模拟浏览器访问京东招聘
headers = {
    "Accept": "*/*",
    "Accept-Encoding": "gzip, deflate, br, zstd",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
    "Origin": "https://zhaopin.jd.com",
    "Pragma": "no-cache",
    "Referer": "https://zhaopin.jd.com/web/job_info_list/3?isHunterFlag=false",
    "Sec-Ch-Ua": '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
    "Sec-Ch-Ua-Mobile": "?0",
    "Sec-Ch-Ua-Platform": '"Windows"',
    "Sec-Fetch-Dest": "empty",
    "Sec-Fetch-Mode": "cors",
    "Sec-Fetch-Site": "same-origin",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
    "X-Requested-With": "XMLHttpRequest"
}

# 登录Cookie，保持会话状态
cookie_str = ('__jdu=17603403343142008747562; wlfstk_smdl=ub60hbvf0g2s8u5pyxjns2fb2l03gtvb; '
              'TrackID=1P4oDr4XPqwWzKwDWRu_4_UeOP6RnR4vGrsonKEeuEFZ89h3hL-1lsc5zllxYrBV6BhYxoo1nteMl5lLtIivBbVGDNM-dumEFbaafdxAD3Fo; '
              'thor=48A7AE1638066EA15412CB5251AEFC8107ABEA59BE4DD47C089393E91C56D975DD386AA83445D095271A06E91F6753A4EEBD41B4F7139D275E6095C88581B0A7D72022FB17DBF1F98F8E3C0D2B4042A28C10C72A2C585175B3F6BC568AC7F7C29A871F6356377A88EDE3B99DE048C52F959125F4DFD8E143FFE614221BAE4023630B1443AABDCA4052E0B2F67197B3E61C69EC748E0D5C3EB8B58C637AEEEF72; '
              'light_key=AASBKE7rOxgWQziEhC_QY6ya123kIBM_FU4iYdzu87Ek3y3AV9F3jFJOmjWGUk2f5PR2vrMW; '
              'pinId=C_LZUcdw8CuA_WidPC2E7w; pin=jd_VHpZKUJPelsF; unick=%E7%84%B6%E5%90%8E%E5%AE%87%E4%BD%A0%E7%9B%B8%E7%88%B1; '
              'ceshi3.com=000; _tp=CxbMgt%2F7gka5zyOGDzDRag%3D%3D; _pst=jd_VHpZKUJPelsF; '
              'zp.sso.tag=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiJsb2dpbkJ5T2xkIiwiaWF0IjoxNzYwNTA3NTI5LCJleHAiOjE3NjMwOTk1Mjl9.xLvCNXREwucALBEzwF0jhkhtUbs5D1RKF-uXNY5jENMdUyROG-rMG2M4nW4ORfzN5Skia1IhotK2d0taowJv6w; '
              'unpl=JF8EAJtnNSttCkxXAx9SGxMRTQ5XW15fGx8LPTMMV1teHFwNGQcbGhR7XlVdWRRLFh9vYxRUWlNJVA4bAysSEHtdVV9fCUgRAWlgNWRaWEIZRElPKxEQe11Vbl0OTBYBamYHVlhaS1EGGgMSFBVNWVNdbQl7EANmVzVkXGhKZAQrSXUTXUtbU19fDUoVAWplBVFeWUpdAx4EHxUTe1xkXQ; '
              '__jdv=29846306|lianmeng__10__www.google.com.hk|t_1003027376_|tuiguang|c6275a1007b242fa99ce8277f9934195|1760517069963; '
              '3AB9D23F7A4B3CSS=jdd03LVXW7AQQE26DOABPQYAY3EB6DJSYADSWLWLTOHYDBJGHCUM25J27NOSGZVJ2J7V3QDYFLWKJ4NQRRH5PMSF2RI4F5YAAAAMZ437J7DIAAAAACHDPQLVTWZZ3RIX; '
              'shshshfpa=cefa655f-9611-dcc5-f04f-0d8be1ec8ad3-1756955929; shshshfpx=cefa655f-9611-dcc5-f04f-0d8be1ec8ad3-1756955929; '
              'flash=3_bhTmbWMtFMJfVp-BLo4k-ea4GVqbGnHBOtoZ5LkZEy6YPePdfLKfsIgUsHIJd1xFCCEEcUuxKBBoqPrxmykb3k7uaNtXC7Z1jn2gLb5BwoNiP6TV8Dp9mGTxtG9Dn9JG-hZAUa6jij9vCh1PYvsQN21NjlArUn1gSr356w_HCZY6PR5fsRJi; '
              'areaId=1; cn=0; shshshfpb=BApXSICP25fxADMtPZFYycUKGZPxaKjsJBhTUXgZp9xJ1MqY-pI62; '
              'ipLoc-djd=1-72-55653-0; 3AB9D23F7A4B3C9B=LVXW7AQQE26DOABPQYAY3EB6DJSYADSWLWLTOHYDBJGHCUM25J27NOSGZVJ2J7V3QDYFLWKJ4NQRRH5PMSF2RI4F5Y; '
              'sdtoken=AAbEsBpEIOVjqTAKCQtvQu17U36gG_rDiFgjEQvxDFInMeUOlnYGCFauvg-tNlP6uaZN4laBI8yNz85SBVKxd4YWIBo0ryEYCi80bbRO9W700UGvEMfO8Dk24zjLodef1Ya5g3YGMPtOnrnFTTD44HOTP4-ZHrpcRFdPVC-prb9u5Q; '
              '__jda=176729966.17603403343142008747562.1760340334.1760516250.1760517070.4; '
              '__jdb=176729966.3.17603403343142008747562|4.1760517070; __jdc=176729966; '
              'JSESSIONID=E058CC60933C75F58ADAE7FC146CD0DD.s1')

headers["Cookie"] = cookie_str

# 请求京东招聘接口，获取职位JSON数据
def get_jd_job_json(url, curPage):
    # 构造分页请求参数
    payload_dict = {
        "pageIndex": curPage,
        "pageSize": 100,
        "workCityJson": "[]",
        "jobTypeJson": "[]",
        "jobSearch": ""
    }
    payload = urlencode(payload_dict)
    
    ner_logger.info(f"准备发送请求到 {url}，页码: {curPage}，请求体: {payload}")

    # 发送POST请求
    with requests.Session() as s:
        resp = s.post(url, data=payload, headers=headers, timeout=15)
        ner_logger.info(f"收到响应，状态码: {resp.status_code}，响应头: {dict(resp.headers)}")
        # 解析返回的JSON数据
        try:
            if resp.status_code == 200:
                json_data = resp.json()
                ner_logger.info(f"成功解析JSON数据，数据结构: {type(json_data)}，数据预览: {str(json_data)[:500]}")
                # 直接返回列表
                if isinstance(json_data, list):
                    data = json_data
                    total = len(data)
                    ner_logger.info(f"返回的是职位列表，数据量: {total}")
                    return True, data, total
                # 包含 data 字段
                elif 'data' in json_data:
                    data = json_data.get('data', [])
                    total = json_data.get('total', len(data))
                    ner_logger.info(f"返回的是包含data字段的结构，数据量: {len(data)}，总数: {total}")
                    return True, data, total
                # 包含 results 字段
                elif 'results' in json_data:
                    data = json_data.get('results', [])
                    total = json_data.get('count', len(data))
                    ner_logger.info(f"返回的是包含results字段的结构，数据量: {len(data)}，总数: {total}")
                    return True, data, total
                # 其他格式
                else:
                    data = [json_data] if not isinstance(json_data, list) else json_data
                    total = len(data)
                    ner_logger.info(f"返回的是其他格式的数据，数据量: {total}")
                    return True, data, total
            else:
                ner_logger.error(f"请求失败，状态码: {resp.status_code}，响应内容: {resp.text}")
                return False, [], 0
        except Exception as e:
            ner_logger.error(f"JSON解析错误: {str(e)}，响应内容: {resp.text}")
            return False, [], 0

# 根据职位信息生成可视化HTML详情页
def generate_job_html(item, tmp_file):
    try:
        ner_logger.info(f"开始生成HTML文件，职位信息: {item}")
        # 提取职位关键字段
        job_name = item.get('positionNameOpen', item.get('positionName', item.get('name', '职位名称')))
        publish_time = item.get('formatPublishTime', item.get('publishTime', ''))
        work_city = item.get('workCity', '')
        job_type = item.get('jobType', '')
        department = item.get('positionDeptName', item.get('departmentName', ''))
        work_content = item.get('workContent', '暂无职位描述信息')
        qualification = item.get('qualification', '暂无职位要求信息')
        
        ner_logger.info(f"职位信息提取结果 - 名称: {job_name}, 发布时间: {publish_time}, 工作城市: {work_city}")
        
        # 拼接HTML内容
        html_content = f"""
<!DOCTYPE html>
<html lang="zh-CN">
<head>
    
    
    <title>{job_name}</title>
    
        body {{ font-family: Arial, sans-serif; margin: 20px; }}
        .job-header {{ border-bottom: 1px solid #eee; padding-bottom: 15px; margin-bottom: 20px; }}
        .job-title {{ font-size: 24px; font-weight: bold; color: #333; }}
        .job-meta {{ margin: 10px 0; }}
        .job-meta span {{ margin-right: 20px; color: #666; }}
        .job-section {{ margin-bottom: 20px; }}
        .section-title {{ font-size: 18px; font-weight: bold; margin-bottom: 10px; color: #444; }}
        .job-description {{ line-height: 1.6; white-space: pre-wrap; }}
    
</head>
<body>
    <div class="job-header">
        <div class="job-title">{job_name}</div>
        <div class="job-meta">
            <span>发布时间: {publish_time}</span>
            <span>工作地点: {work_city}</span>
            <span>职位类型: {job_type}</span>
        </div>
        <div class="job-meta">
            <span>所属部门: {department}</span>
        </div>
    </div>
    
    <div class="job-section">
        <div class="section-title">职位描述</div>
        <div class="job-description">
            {work_content}
        </div>
    </div>
    
    <div class="job-section">
        <div class="section-title">职位要求</div>
        <div class="job-description">
            {qualification}
        </div>
    </div>
</body>
</html>
        """
        
        # 写入HTML文件
        with open(tmp_file, "w", encoding="utf-8") as f:
            f.write(html_content)
        ner_logger.info(f"成功生成HTML文件: {tmp_file}")
            
        time.sleep(1)
        return True
    except Exception as e:
        ner_logger.error(f"生成HTML文件失败：{e}")
        return False

# 将京东原始数据转为统一标准JSON
def transform_job_json(item, job_type, channel, target_url, tmp_file, json_file):
    ner_logger.info(f"开始转换JSON数据，原始数据: {item}")
    # 字段映射
    field_mapping = {
        "announcement_name": "positionNameOpen",
        "publish_time": "formatPublishTime",
        "hd_dept": "positionDeptName",
        "hd_loc": "workCity",
        "hd_job_num": "",
        "hd_job_category": "jobType"
    }
    
    # 适配不同返回格式
    if "positionNameOpen" not in item:
        if "formatPublishTime" in item:
            field_mapping = {
                "announcement_name": "name",
                "publish_time": "formatPublishTime",
                "hd_dept": "departmentName",
                "hd_loc": "workCity",
                "hd_job_num": "",
                "hd_job_category": "jobTypeName"
            }
        elif "publishTime" in item:
            field_mapping = {
                "announcement_name": "name",
                "publish_time": "publishTime",
                "hd_dept": "departmentName",
                "hd_loc": "workCity",
                "hd_job_num": "",
                "hd_job_category": "jobTypeName"
            }

    # 固定公共字段
    fixed_fields = {
        "link": target_url,
        "full_url": target_url,
        "last_url": target_url,
        "file_path": tmp_file,
        "parent_url": "https://zhaopin.jd.com/",
        "channel": channel,
        "job_type": job_type
    }
    
    target_json = {}
    # 字段映射赋值
    for target_field, source_field in field_mapping.items():
        if source_field:
            target_json[target_field] = item.get(source_field, "")
        else:
            target_json[target_field] = ""
    
    # 写入固定字段
    target_json.update(fixed_fields)
    
    ner_logger.info(f"转换后的JSON数据: {target_json}")
    
    # 保存标准JSON
    with open(json_file, 'w', encoding='utf-8') as f:
        json.dump(target_json, f, ensure_ascii=False, indent=4)
        ner_logger.info(f"成功保存JSON文件: {json_file}")
        time.sleep(1)

# 京东招聘主爬取逻辑
def api_proc_jd(spider_com, _key, com_info, k, url, _stat):
    ner_logger.info(f"开始处理京东招聘数据，参数 - _key: {_key}, k: {k}, url: {url}, _stat: {_stat}")
    # 未传入URL则使用默认接口
    if not url or url == "":
        url = "https://zhaopin.jd.com/web/job/job_list"
        ner_logger.info(f"URL为空，使用默认URL: {url}")

    # 判断社招/校招
    job_type = "shezhao"
    
    if k.startswith("shezhao"):
        job_type = "shezhao"
        ner_logger.info("设置job_type为shezhao")
    elif k.startswith("xiaozhao"):
        job_type = "xiaozhao"
        ner_logger.info("设置job_type为xiaozhao")

    # 获取临时文件目录
    key_tmp_dir = spider_com.get_key_dir(_key)
    ner_logger.info(f"使用临时目录: {key_tmp_dir}")
    
    total_page = 50
    
    # 循环翻页爬取
    for curPage in range(1, 100):
        ner_logger.info(f"开始处理第 {curPage} 页")
        flag, json_data, totalcount = get_jd_job_json(url, curPage)
        if flag:
            ner_logger.info("jd json response: total_page=%s, data_count=%s", total_page, len(json_data))
            if total_page == 0:
                total_page = int(totalcount / 100) + 1
                ner_logger.info(f"计算总页数: {total_page} (总数据量: {totalcount})")
                
            # 保存列表页JSON
            _hash = hashlib.md5(url.encode("utf-8")).hexdigest()
            tmp_fname = f'{key_tmp_dir}/index_{_hash}_{curPage}.json'
            with open(tmp_fname, 'w', encoding='utf-8') as f:
                json.dump(json_data, f, ensure_ascii=False, indent=4)
            ner_logger.info(f"保存页面JSON数据到: {tmp_fname}")
                
            # 终止条件
            if curPage >= total_page:
                ner_logger.info("已达到总页数，结束循环")
                break
            if curPage > 5 and _stat.get('method', '') != "cp_full":
                ner_logger.info("页数超过5页且method不为cp_full，结束循环")
                break
                
            # 遍历职位生成详情
            for idx, item in enumerate(json_data):
                ner_logger.info(f"处理第 {curPage} 页中的第 {idx+1} 个项目")
                job_id = item.get("id", "")
                _fullurl = f"https://zhaopin.jd.com/web/job_info_list/3"
                
                # 生成文件路径
                _hash = hashlib.md5((_fullurl + str(job_id)).encode("utf-8")).hexdigest()
                tmp_file = os.path.join(key_tmp_dir, f"detail_{_hash}.html")
                tmp_json_file = os.path.join(key_tmp_dir, f"detail_{_hash}.json")
                
                # 生成HTML
                html_result = generate_job_html(item, tmp_file)
                if not html_result:
                    ner_logger.error(f"生成HTML文件失败，跳过当前项目: {item}")
                    continue
                
                # 转换标准JSON
                transform_job_json(item, job_type, _key, _fullurl, tmp_file, tmp_json_file)
                time.sleep(1)
        else:
            ner_logger.error(f"获取第 {curPage} 页数据失败")
            if curPage == 1:
                ner_logger.error("第一页数据获取失败，终止执行")
                return False
        time.sleep(1)
        
    ner_logger.info("京东招聘数据处理完成")
    return True

项目分区导航：isoftstone_data_proc_api ⬅️ | 03-jd_data_proc_api | ➡️ kingdee_data_proc_api