commit

patterns.json

@@ -1,380 +1,406 @@
 [
-    {
-    "site": "Guosen Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关研究报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
-        "date_format": "%Y-%m-%d",
-        "split":[
-                  {
-                    "string": "-",
-                    "index": -1
-                  }
-                ]
-    },
-    {
-        "site": "Soochow Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 2,
-        "keyword": "相关研究",
-        "article_regex": "《(.*?)》",
-        "date_regex": "\\b\\d{4}-\\d{2}-\\d{2}|\\d{4} -\\d{2}-\\d{2}\\b",
-        "date_format": "%Y-%m-%d",
-        "split":[
-                  {
-                    "string": "-",
-                    "index": 0
-                  },
-                  {
-                    "string": "—",
-                    "index": 0
-                  }
-                ]
-    },
-    {
-        "site": "BOCI Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关研究报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "20\\d{6}|20\\d{5}\\s{1}\\d{1}",
-        "date_format": "%Y%m%d"
-    },
-    {
-        "site": "Tianfeng Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 3,
-        "keyword": "相关报告",
-        "article_regex": " 《(.*?)》",
-        "date_regex": "\\b\\d{4}-\\d{2}-\\d{2}|\\d{4} -\\d{2}-\\d{2}\\b",
-        "date_format": "%Y-%m-%d",
-        "remove": ["宏观报告：", "宏观-", "宏观报告-", "——"],
-        "split":[
-                  {
-                    "string": "：",
-                    "index": 1
-                  },
-                  {
-                    "string": "-",
-                    "index": 0
-                  },
-                  {
-                    "string": "(",
-                    "index": 1
-                  }
-                ]
-    },
-    {
-        "site": "Kaiyuan Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": " ",
-        "article_regex": " 《(.*?)》",
-        "date_regex": "\\b\\d{4}\\.\\d{1,2}\\.\\d{1,2}\\b",
-        "date_format": "%Y.%m.%d",
-        "split":[
-                  {
-                    "string": "—",
-                    "index": 1
-                  }
-                ]
-    },
-    {
-        "site": "Huafu Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 4,
-        "keyword": "相关报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "20\\d{2}\\s?\\.\\s?\\d{1}\\s?\\d{1}\\s?\\.\\s?\\d{1,2}",
-        "date_format": "%Y.%m.%d",
-        "split":[
-                  {
-                    "string": "：",
-                    "index": 1
-                  },
-                  {
-                    "string": "——",
-                    "index": 0
-                  }
-                ]
-    },
-    {
-        "site": "Minsheng Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关研究",
-        "article_regex": "\\.(.*?)\\-",
-        "date_regex": "20\\d{2}\\/\\d{2}\\/\\d{2}",
-        "date_format": "%Y/%m/%d",
-        "split":[
-                  {
-                    "string": "：",
-                    "index": 1
-                  }
-                ]
-    },
-    {
-        "site": "Guolian Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关报告 ",
-        "article_regex": "《(.*?)》",
-        "date_regex": "[》 ]20\\d{2}\\.\\d{2}\\.\\d{2}",
-        "date_format": "%Y.%m.%d",
-        "split":[
-                  {
-                    "string": "：",
-                    "index": 0
-                  }
-                ]
-    },
-    {
-        "site": "Southwest Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关研究",
-        "article_regex": "\\.(.*?)\\(",
-        "date_regex": "(20\\d{2}\\s?-\\d{2}\\-\\d{2})",
-        "date_format": "%Y-%m-%d"
-    },
-    {
-        "site": "Guangdong Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "近期报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "20\\d{2}\\s?-\\d{2}\\-\\d{2}",
-        "date_format": "%Y-%m-%d"
-    },
-    {
-        "site": "China Post Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "近期研究报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "20\\d{2}\\s?.\\d{2}\\.\\d{2}",
-        "date_format": "%Y.%m.%d",
-        "split":[
-                  {
-                    "string": "-",
-                    "index": 1
-                  },
-                  {
-                    "string": "——",
-                    "index": 0
-                  }
-                ]
-    },
-    {
-        "site": "Shanxi Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": " ",
-        "article_regex": "】(.*?)\\（",
-        "date_regex": "20\\d{2}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2} ",
-        "date_format": "%Y.%m.%d"
-    },
-    {
-        "site": "Shanghai Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "Table_Rep",
-        "article_regex": "《(.*?)》",
-        "date_regex": "20\\d{2}年\\d{2}月\\d{2}",
-        "date_format": "%Y年%m月%d"
-    },
-    {
-        "site": "Guoyuan Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 2,
-        "keyword": "[Table_Report]",
-        "article_regex": "《(.*?)》 ",
-        "date_regex": " 20\\d{2}.\\d{2}.\\d{2} ",
-        "date_format": "%Y.%m.%d",
-        "split":[
-                  {
-                    "string": "：",
-                    "index": 0
-                  }
-                ]
-    },
-    {
-        "site": "Mago Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关研究",
-        "article_regex": "《(.*?)》",
-        "date_regex": "20\\d{2}\\s?.\\s?\\d{2}\\s?.\\s?\\d{2} ",
-        "date_format": "%Y.%m.%d",
-        "split":[
-                  {
-                    "string": "（",
-                    "index": 0
-                  }
-                ]
-    },
-    {
-        "site": "Fed Securities, Inc.",
-        "pages": [0],
-        "date_range": 3,
-        "keyword": "相关报告",
-        "article_regex": "：(.*?)20",
-        "date_regex": "20\\d{2}\\s?.\\s?\\d{2}\\s?.\\s?\\d{2}",
-        "date_format": "%Y.%m.%d"
-    },
-    {
-        "site": "Huabao Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关研究报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
-        "date_format": "%Y-%m-%d"
-    },
-    {
-        "site": "Ruitingdog (Shenzhen) Information Technology Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "近期研究",
-        "article_regex": "：(.*?)-",
-        "date_regex": "\\d{4}\\s?/\\s?\\d{1,2}\\s?/\\s?\\d{1,2}",
-        "date_format": "%Y/%m/%d"
-    },
-    {
-        "site": "Oriental Fortune Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关研究",
-        "article_regex": "《(.*?)》",
-        "date_regex": "\\d{4}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2}",
-        "date_format": "%Y.%m.%d"
-    },
-    {
-        "site": "Yongxing Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关报告：",
-        "article_regex": "《(.*?)》",
-        "date_regex": "—— \\d{4}\\s?年\\s?\\d{1,2}\\s?月\\s?\\d{1,2}",
-        "date_format": "——%Y年%m月%d"
-    },
-    {
-        "site": "Minmetals Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关研究",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(d{4}\\s/\\d{2}/\\d{2}) ",
-        "date_format": "(%Y/%m/%d) "
-    },
-    {
-        "site": "Hualong Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关阅读",
-        "article_regex": "《(.*?)》",
-        "date_regex": "\\d{4}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2}",
-        "date_format": "%Y.%m.%d"
-    },
-    {
-        "site": "Hebei Yuanda Information Technology Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关报告：",
-        "article_regex": "《(.*?)》",
-        "date_regex": "\\d{4}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2}",
-        "date_format": "%Y.%m.%d"
-    },
-    {
-        "site": "Huaxin Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关研究",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
-        "date_format": "%Y-%m-%d"
-    },
-    {
-        "site": "Far East Credit Rating Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "1.",
-        "article_regex": "《(.*?)》",
-        "date_regex": "\\d{4}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2}",
-        "date_format": "%Y.%m.%d"
-    },
-    {
-        "site": "Beijing Tengjing Big Data Application Technology Research Institute",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
-        "date_format": "%Y-%m-%d"
-    },
-    {
-        "site": "Wanhe Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
-        "date_format": "%Y-%m-%d"
-    },
-    {
-        "site": "Centaline Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(\\d{4}\\s?-\\s?\\d{2}\\s?-\\s?\\d{2})",
-        "date_format": "%Y-%m-%d"
-    },
-    {
-        "site": "Tengjing Digital Research",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(\\d{4}\\s?-\\s?\\d{2}\\s?-\\s?\\d{2})",
-        "date_format": "%Y-%m-%d"
-    },
-    {
-        "site": "Guoyuan Securities",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关研究报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "\\d{4}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2}",
-        "date_format": "%Y.%m.%d"
-    },
-    {
-        "site": "China Galaxy Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(\\d{4}\\s?-\\s?\\d{2}\\s?-\\s?\\d{2})",
-        "date_format": "%Y-%m-%d"
-    },
-    {
-        "site": "Shengang Securities Co., Ltd.",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(\\d{4}\\s?-\\s?\\d{2}\\s?-\\s?\\d{2})",
-        "date_format": "%Y-%m-%d"
-    },
-    {
-        "site": "SDIC Anxin Futures",
-        "pages": [0],
-        "date_range": 1,
-        "keyword": "相关报告",
-        "article_regex": "《(.*?)》",
-        "date_regex": "(\\d{4}\\s?-\\s?\\d{2}\\s?-\\s?\\d{2})",
-        "date_format": "%Y-%m-%d"
-    }
+  {
+      "site": "China Development Bank Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告",
+      "article_regex": "《(.*?)》",
+      "no_date": 1,
+      "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
+      "date_format": "%Y-%m-%d",
+      "split":[
+                {
+                  "string": "—",
+                  "index": -1
+                }
+              ]
+  },
+  {
+  "site": "Donghai Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究",
+      "article_regex": "《(.*?)》",
+      "no_date": 1,
+      "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+  "site": "Guosen Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
+      "date_format": "%Y-%m-%d",
+      "split":[
+                {
+                  "string": "-",
+                  "index": -1
+                }
+              ]
+  },
+  {
+      "site": "Soochow Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 2,
+      "keyword": "相关研究",
+      "article_regex": "《(.*?)》",
+      "date_regex": "\\b\\d{4}-\\d{2}-\\d{2}|\\d{4} -\\d{2}-\\d{2}\\b",
+      "date_format": "%Y-%m-%d",
+      "split":[
+                {
+                  "string": "-",
+                  "index": 0
+                },
+                {
+                  "string": "—",
+                  "index": 0
+                }
+              ]
+  },
+  {
+      "site": "BOCI Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "20\\d{6}|20\\d{5}\\s{1}\\d{1}",
+      "date_format": "%Y%m%d"
+  },
+  {
+      "site": "Tianfeng Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 3,
+      "keyword": "相关报告",
+      "article_regex": " 《(.*?)》",
+      "date_regex": "\\b\\d{4}-\\d{2}-\\d{2}|\\d{4} -\\d{2}-\\d{2}\\b",
+      "date_format": "%Y-%m-%d",
+      "remove": ["宏观报告：", "宏观-", "宏观报告-", "——"],
+      "split":[
+                {
+                  "string": "：",
+                  "index": 1
+                },
+                {
+                  "string": "-",
+                  "index": 0
+                },
+                {
+                  "string": "(",
+                  "index": 1
+                }
+              ]
+  },
+  {
+      "site": "Kaiyuan Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": " ",
+      "article_regex": " 《(.*?)》",
+      "date_regex": "\\b\\d{4}\\.\\d{1,2}\\.\\d{1,2}\\b",
+      "date_format": "%Y.%m.%d",
+      "split":[
+                {
+                  "string": "—",
+                  "index": 1
+                }
+              ]
+  },
+  {
+      "site": "Huafu Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 4,
+      "keyword": "相关报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "20\\d{2}\\s?\\.\\s?\\d{1}\\s?\\d{1}\\s?\\.\\s?\\d{1,2}",
+      "date_format": "%Y.%m.%d",
+      "split":[
+                {
+                  "string": "：",
+                  "index": 1
+                },
+                {
+                  "string": "——",
+                  "index": 0
+                }
+              ]
+  },
+  {
+      "site": "Minsheng Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究",
+      "article_regex": "\\.(.*?)\\-",
+      "date_regex": "20\\d{2}\\/\\d{2}\\/\\d{2}",
+      "date_format": "%Y/%m/%d",
+      "split":[
+                {
+                  "string": "：",
+                  "index": 1
+                }
+              ]
+  },
+  {
+      "site": "Guolian Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告 ",
+      "article_regex": "《(.*?)》",
+      "date_regex": "[》 ]20\\d{2}\\.\\d{2}\\.\\d{2}",
+      "date_format": "%Y.%m.%d",
+      "split":[
+                {
+                  "string": "：",
+                  "index": 0
+                }
+              ]
+  },
+  {
+      "site": "Southwest Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究",
+      "article_regex": "\\.(.*?)\\(",
+      "date_regex": "(20\\d{2}\\s?-\\d{2}\\-\\d{2})",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+      "site": "Guangdong Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "近期报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "20\\d{2}\\s?-\\d{2}\\-\\d{2}",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+      "site": "China Post Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "近期研究报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "20\\d{2}\\s?.\\d{2}\\.\\d{2}",
+      "date_format": "%Y.%m.%d",
+      "split":[
+                {
+                  "string": "-",
+                  "index": 1
+                },
+                {
+                  "string": "——",
+                  "index": 0
+                }
+              ]
+  },
+  {
+      "site": "Shanxi Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": " ",
+      "article_regex": "】(.*?)\\（",
+      "date_regex": "20\\d{2}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2} ",
+      "date_format": "%Y.%m.%d"
+  },
+  {
+      "site": "Shanghai Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "Table_Rep",
+      "article_regex": "《(.*?)》",
+      "date_regex": "20\\d{2}年\\d{2}月\\d{2}",
+      "date_format": "%Y年%m月%d"
+  },
+  {
+      "site": "Guoyuan Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 2,
+      "keyword": "[Table_Report]",
+      "article_regex": "《(.*?)》 ",
+      "date_regex": " 20\\d{2}.\\d{2}.\\d{2} ",
+      "date_format": "%Y.%m.%d",
+      "split":[
+                {
+                  "string": "：",
+                  "index": 0
+                }
+              ]
+  },
+  {
+      "site": "Mago Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究",
+      "article_regex": "《(.*?)》",
+      "date_regex": "20\\d{2}\\s?.\\s?\\d{2}\\s?.\\s?\\d{2} ",
+      "date_format": "%Y.%m.%d",
+      "split":[
+                {
+                  "string": "（",
+                  "index": 0
+                }
+              ]
+  },
+  {
+      "site": "Fed Securities, Inc.",
+      "pages": [0],
+      "date_range": 3,
+      "keyword": "相关报告",
+      "article_regex": "：(.*?)20",
+      "date_regex": "20\\d{2}\\s?.\\s?\\d{2}\\s?.\\s?\\d{2}",
+      "date_format": "%Y.%m.%d"
+  },
+  {
+      "site": "Huabao Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+      "site": "Ruitingdog (Shenzhen) Information Technology Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "近期研究",
+      "article_regex": "：(.*?)-",
+      "date_regex": "\\d{4}\\s?/\\s?\\d{1,2}\\s?/\\s?\\d{1,2}",
+      "date_format": "%Y/%m/%d"
+  },
+  {
+      "site": "Oriental Fortune Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究",
+      "article_regex": "《(.*?)》",
+      "date_regex": "\\d{4}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2}",
+      "date_format": "%Y.%m.%d"
+  },
+  {
+      "site": "Yongxing Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告：",
+      "article_regex": "《(.*?)》",
+      "date_regex": "—— \\d{4}\\s?年\\s?\\d{1,2}\\s?月\\s?\\d{1,2}",
+      "date_format": "——%Y年%m月%d"
+  },
+  {
+      "site": "Minmetals Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(d{4}\\s/\\d{2}/\\d{2}) ",
+      "date_format": "(%Y/%m/%d) "
+  },
+  {
+      "site": "Hualong Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关阅读",
+      "article_regex": "《(.*?)》",
+      "date_regex": "\\d{4}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2}",
+      "date_format": "%Y.%m.%d"
+  },
+  {
+      "site": "Hebei Yuanda Information Technology Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告：",
+      "article_regex": "《(.*?)》",
+      "date_regex": "\\d{4}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2}",
+      "date_format": "%Y.%m.%d"
+  },
+  {
+      "site": "Huaxin Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+      "site": "Far East Credit Rating Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "1.",
+      "article_regex": "《(.*?)》",
+      "date_regex": "\\d{4}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2}",
+      "date_format": "%Y.%m.%d"
+  },
+  {
+      "site": "Beijing Tengjing Big Data Application Technology Research Institute",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+      "site": "Wanhe Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(\\d{4}-\\d{2}-\\d{2})",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+      "site": "Centaline Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(\\d{4}\\s?-\\s?\\d{2}\\s?-\\s?\\d{2})",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+      "site": "Tengjing Digital Research",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(\\d{4}\\s?-\\s?\\d{2}\\s?-\\s?\\d{2})",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+      "site": "Guoyuan Securities",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关研究报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "\\d{4}\\s?.\\s?\\d{1,2}\\s?.\\s?\\d{1,2}",
+      "date_format": "%Y.%m.%d"
+  },
+  {
+      "site": "China Galaxy Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(\\d{4}\\s?-\\s?\\d{2}\\s?-\\s?\\d{2})",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+      "site": "Shengang Securities Co., Ltd.",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(\\d{4}\\s?-\\s?\\d{2}\\s?-\\s?\\d{2})",
+      "date_format": "%Y-%m-%d"
+  },
+  {
+      "site": "SDIC Anxin Futures",
+      "pages": [0],
+      "date_range": 1,
+      "keyword": "相关报告",
+      "article_regex": "《(.*?)》",
+      "date_regex": "(\\d{4}\\s?-\\s?\\d{2}\\s?-\\s?\\d{2})",
+      "date_format": "%Y-%m-%d"
+  }
 ]

utils.py

@@ -123,41 +123,67 @@ def extract_reference(row):
         if 'remove' in pattern:
             for remove_string in pattern['remove']:
                 reference_titles = [s.replace(remove_string, '') for s in reference_titles]
-        for title, date in zip(reference_titles, reference_dates):
-            try:
-                date = datetime.strptime(date, pattern['date_format'])
-            except:
-                date = datetime(2006, 1, 1)
-            dates = []
-            if 'date_range' in pattern:
-                for i in range(pattern['date_range'] + 1):
-                    dates.append((date + timedelta(days=i)).strftime('%Y-%m-%d'))
-                    dates.append((date - timedelta(days=i)).strftime('%Y-%m-%d'))
-            dates.append(date.strftime('%Y-%m-%d'))
-            date = date.strftime('%Y-%m-%d')
-            if 'split' in pattern:
-                for split_item in pattern['split']:
-                    if 'exceptional_string' in split_item:
-                        if split_item['string'] in title and isnot_substring(split_item['exceptional_string'], title):
-                            title = re.split(split_item['string'], title)[split_item['index']]
-                    else:
-                        if split_item['string'] in title:
-                            title = title.split(split_item['string'])[split_item['index']]
-            if len(data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]) == 0:
-                print("------------ = 0 ------------")
-                print(date, repr(title))
-            elif len(data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]) > 1:
-                print("------------ > 1 ------------")
-                print(date, repr(title))
-            else:
-                print("------------ = 1 ------------")
-                reference_df = data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]
-                row['referenceID'] = reference_df.iloc[0]['id']
-                row['link'] = reference_df.iloc[0]['link']
-                row['sourceID'] = row['id']
-                row['refID'] = uuid.uuid5(uuid.NAMESPACE_OID, str(row['sourceID'])+str(row['referenceID']))
-                print(date, repr(title), row['sourceID'],row['referenceID'])
-                update_reference(row)
+        if len(reference_dates) > 0:
+            for title, date in zip(reference_titles, reference_dates):
+                try:
+                    date = datetime.strptime(date, pattern['date_format'])
+                except:
+                    date = datetime(2006, 1, 1)
+                dates = []
+                if 'date_range' in pattern:
+                    for i in range(pattern['date_range'] + 1):
+                        dates.append((date + timedelta(days=i)).strftime('%Y-%m-%d'))
+                        dates.append((date - timedelta(days=i)).strftime('%Y-%m-%d'))
+                dates.append(date.strftime('%Y-%m-%d'))
+                date = date.strftime('%Y-%m-%d')
+                if 'split' in pattern:
+                    for split_item in pattern['split']:
+                        if 'exceptional_string' in split_item:
+                            if split_item['string'] in title and isnot_substring(split_item['exceptional_string'], title):
+                                title = re.split(split_item['string'], title)[split_item['index']]
+                        else:
+                            if split_item['string'] in title:
+                                title = title.split(split_item['string'])[split_item['index']]
+                if len(data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]) == 0:
+                    print("------------ = 0 ------------")
+                    print(date, repr(title))
+                elif len(data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]) > 1:
+                    print("------------ > 1 ------------")
+                    print(date, repr(title))
+                else:
+                    print("------------ = 1 ------------")
+                    reference_df = data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site']) & (data['publishdate'].isin(dates))]
+                    row['referenceID'] = reference_df.iloc[0]['id']
+                    row['link'] = reference_df.iloc[0]['link']
+                    row['sourceID'] = row['id']
+                    row['refID'] = uuid.uuid5(uuid.NAMESPACE_OID, str(row['sourceID'])+str(row['referenceID']))
+                    print(date, repr(title), row['sourceID'],row['referenceID'])
+                    update_reference(row)
+        else:
+            for title in reference_titles:
+                if 'split' in pattern:
+                    for split_item in pattern['split']:
+                        if 'exceptional_string' in split_item:
+                            if split_item['string'] in title and isnot_substring(split_item['exceptional_string'], title):
+                                title = re.split(split_item['string'], title)[split_item['index']]
+                        else:
+                            if split_item['string'] in title:
+                                title = title.split(split_item['string'])[split_item['index']]
+                if len(data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site'])]) == 0:
+                    print("------------ = 0 ------------")
+                    print(repr(title))
+                elif len(data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site'])]) > 1:
+                    print("------------ > 1 ------------")
+                    print(repr(title))
+                else:
+                    print("------------ = 1 ------------")
+                    reference_df = data[(data['titleCN'].str.contains(title)) & (data['site'] == row['site'])]
+                    row['referenceID'] = reference_df.iloc[0]['id']
+                    row['link'] = reference_df.iloc[0]['link']
+                    row['sourceID'] = row['id']
+                    row['refID'] = uuid.uuid5(uuid.NAMESPACE_OID, str(row['sourceID'])+str(row['referenceID']))
+                    print(repr(title), row['sourceID'],row['referenceID'])
+                    update_reference(row)
     except Exception as error:
         print(error)