RSS Data Crawler #399
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: RSS Data Crawler | |
| on: | |
| schedule: | |
| - cron: "0 22 * * *" # 한국 시간 오전 7시 | |
| - cron: "0 10 * * *" # 한국 시간 오후 7시 | |
| workflow_dispatch: # 수동 실행 가능 | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "scripts/rss-crawler.js" | |
| # 권한 설정 | |
| permissions: | |
| contents: write | |
| issues: write | |
| pull-requests: write | |
| jobs: | |
| crawl-rss: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "18" | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: latest | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Run RSS crawler | |
| run: pnpm run crawl-rss | |
| env: | |
| NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL }} | |
| SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} | |
| FIREBASE_SERVICE_ACCOUNT_KEY: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_KEY }} | |
| FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} | |
| NODE_OPTIONS: "--max-old-space-size=4096" | |
| - name: Upload crawl results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rss-crawl-results | |
| path: rss-crawl-results.json | |
| retention-days: 30 | |
| - name: Create summary comment | |
| if: github.event_name == 'push' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| let results = []; | |
| try { | |
| const data = fs.readFileSync('rss-crawl-results.json', 'utf8'); | |
| results = JSON.parse(data); | |
| } catch (error) { | |
| console.log('결과 파일을 읽을 수 없습니다.'); | |
| } | |
| const successCount = results.filter(r => r.success).length; | |
| const errorCount = results.filter(r => !r.success).length; | |
| const totalCount = results.length; | |
| const newPostsCount = results.reduce((sum, r) => sum + (r.newPosts || 0), 0); | |
| let comment = `## 📊 RSS 크롤링 완료\n\n`; | |
| comment += `📈 **크롤링 결과**: ${successCount}/${totalCount}개 피드 성공\n`; | |
| comment += `🆕 **새 포스트**: ${newPostsCount}개 추가됨\n\n`; | |
| if (errorCount > 0) { | |
| comment += `❌ **실패한 피드**:\n`; | |
| results.filter(r => !r.success).forEach(feed => { | |
| comment += `- \`${feed.url}\`: ${feed.error}\n`; | |
| }); | |
| } | |
| comment += `\n---\n`; | |
| comment += `*이 크롤링은 자동으로 실행됩니다.*`; | |
| // 커밋에 코멘트 추가 (GitHub API 사용) | |
| const { data: commits } = await github.rest.repos.listCommits({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| per_page: 1 | |
| }); | |
| if (commits.length > 0) { | |
| await github.rest.repos.createCommitComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| commit_sha: commits[0].sha, | |
| body: comment | |
| }); | |
| } |