Update templates/index.html

templates/index.html

@@ -43,8 +43,11 @@
             <div class="category-section">
                 <div class="category-title" onclick="toggleCategory('{{ category }}')">{{ category }} <span class="loading-spinner" id="spinner-{{ category }}" style="display: none;"></span></div>
                 <div class="tiles" id="category-{{ category }}" data-last-update="0">
+                    {% set seen_articles = [] %}
                     {% for article in articles %}
-                    <div class="article-tile" data-published="{{ article.published }}" data-id="{{ loop.index }}">
+                    {% set article_key = article.title + '|' + article.link + '|' + article.published %}
+                    {% if article_key not in seen_articles %}
+                    <div class="article-tile" data-published="{{ article.published }}" data-id="{{ loop.index }}" data-key="{{ article_key }}">
                         {% if article.image != "svg" %}
                         <img src="{{ article.image }}" alt="Article Image">
                         {% else %}
@@ -57,6 +60,8 @@
                         <div class="description">{{ article.description }}</div>
                         <div class="published">Published: {{ article.published }}</div>
                     </div>
+                    {% do seen_articles.append(article_key) %}
+                    {% endif %}
                     {% endfor %}
                 </div>
                 <div class="tiles" id="all-{{ category }}" style="display: none;"></div>
@@ -73,6 +78,10 @@
         let lastUpdate = 0;
         let expandedCategories = new Set();
 
+        function getArticleKey(article) {
+            return `${article.title}|${article.link}|${article.published}`;
+        }
+
         function toggleCategory(category) {
             const spinner = document.getElementById(`spinner-${category}`);
             const tilesDiv = document.getElementById(`category-${category}`);
@@ -89,21 +98,28 @@
                         spinner.style.display = 'none';
                         if (data.articles.length > 0) {
                             let html = '';
+                            const seenKeys = new Set();
                             data.articles.sort((a, b) => new Date(b.published) - new Date(a.published));
                             data.articles.forEach((article, index) => {
-                                html += `
-                                    <div class="article-tile" data-published="${article.published}" data-id="${index}">
-                                        ${article.image !== "svg" ? `<img src="${article.image}" alt="Article Image">` : `
-                                            <svg width="100%" height="150" xmlns="http://www.w3.org/2000/svg">
-                                                <rect width="100%" height="100%" fill="#e0e0e0"/>
-                                                <text x="50%" y="50%" text-anchor="middle" dy=".3em" fill="#666">No Image</text>
-                                            </svg>
-                                        `}
-                                        <div class="title"><a href="${article.link}" target="_blank">${article.title}</a></div>
-                                        <div class="description">${article.description}</div>
-                                        <div class="published">Published: ${article.published}</div>
-                                    </div>
-                                `;
+                                const key = getArticleKey(article);
+                                if (!seenKeys.has(key)) {
+                                    seenKeys.add(key);
+                                    html += `
+                                        <div class="article-tile" data-published="${article.published}" data-id="${index}" data-key="${key}">
+                                            ${article.image !== "svg" ? `<img src="${article.image}" alt="Article Image">` : `
+                                                <svg width="100%" height="150" xmlns="http://www.w3.org/2000/svg">
+                                                    <rect width="100%" height="100%" fill="#e0e0e0"/>
+                                                    <text x="50%" y="50%" text-anchor="middle" dy=".3em" fill="#666">No Image</text>
+                                                </svg>
+                                            `}
+                                            <div class="title"><a href="${article.link}" target="_blank">${article.title}</a></div>
+                                            <div class="description">${article.description}</div>
+                                            <div class="published">Published: ${article.published}</div>
+                                        </div>
+                                    `;
+                                } else {
+                                    console.warn(`Duplicate article skipped in category ${category}: ${key}`);
+                                }
                             });
                             tilesDiv.style.display = 'none';
                             allTilesDiv.innerHTML = html;
@@ -134,12 +150,14 @@
                             if (tilesDiv) {
                                 const existingArticles = Array.from(tilesDiv.querySelectorAll('.article-tile'));
                                 let currentIds = new Set(existingArticles.map(a => a.dataset.id));
+                                let currentKeys = new Set(existingArticles.map(a => a.dataset.key));
                                 let newHtml = '';
                                 articles.sort((a, b) => new Date(b.published) - new Date(a.published));
                                 articles.slice(0, 10).forEach((article, index) => {
-                                    if (!currentIds.has(index.toString())) {
+                                    const key = getArticleKey(article);
+                                    if (!currentKeys.has(key)) {
                                         newHtml += `
-                                            <div class="article-tile" data-published="${article.published}" data-id="${index}">
+                                            <div class="article-tile" data-published="${article.published}" data-id="${index}" data-key="${key}">
                                                 ${article.image !== "svg" ? `<img src="${article.image}" alt="Article Image">` : `
                                                     <svg width="100%" height="150" xmlns="http://www.w3.org/2000/svg">
                                                         <rect width="100%" height="100%" fill="#e0e0e0"/>
@@ -151,6 +169,8 @@
                                                 <div class="published">Published: ${article.published}</div>
                                             </div>
                                         `;
+                                    } else {
+                                        console.warn(`Duplicate article skipped in update for ${category}: ${key}`);
                                     }
                                 });
                                 if (newHtml) {
@@ -160,20 +180,27 @@
                                 }
                                 if (allTilesDiv.style.display === 'grid') {
                                     let allNewHtml = '';
+                                    const allSeenKeys = new Set(Array.from(allTilesDiv.querySelectorAll('.article-tile')).map(a => a.dataset.key));
                                     articles.forEach((article, index) => {
-                                        allNewHtml += `
-                                            <div class="article-tile" data-published="${article.published}" data-id="${index}">
-                                                ${article.image !== "svg" ? `<img src="${article.image}" alt="Article Image">` : `
-                                                    <svg width="100%" height="150" xmlns="http://www.w3.org/2000/svg">
-                                                        <rect width="100%" height="100%" fill="#e0e0e0"/>
-                                                        <text x="50%" y="50%" text-anchor="middle" dy=".3em" fill="#666">No Image</text>
-                                                    </svg>
-                                                `}
-                                                <div class="title"><a href="${article.link}" target="_blank">${article.title}</a></div>
-                                                <div class="description">${article.description}</div>
-                                                <div class="published">Published: ${article.published}</div>
-                                            </div>
-                                        `;
+                                        const key = getArticleKey(article);
+                                        if (!allSeenKeys.has(key)) {
+                                            allSeenKeys.add(key);
+                                            allNewHtml += `
+                                                <div class="article-tile" data-published="${article.published}" data-id="${index}" data-key="${key}">
+                                                    ${article.image !== "svg" ? `<img src="${article.image}" alt="Article Image">` : `
+                                                        <svg width="100%" height="150" xmlns="http://www.w3.org/2000/svg">
+                                                            <rect width="100%" height="100%" fill="#e0e0e0"/>
+                                                            <text x="50%" y="50%" text-anchor="middle" dy=".3em" fill="#666">No Image</text>
+                                                        </svg>
+                                                    `}
+                                                    <div class="title"><a href="${article.link}" target="_blank">${article.title}</a></div>
+                                                    <div class="description">${article.description}</div>
+                                                    <div class="published">Published: ${article.published}</div>
+                                                </div>
+                                            `;
+                                        } else {
+                                            console.warn(`Duplicate article skipped in expanded view for ${category}: ${key}`);
+                                        }
                                     });
                                     allTilesDiv.innerHTML = (Array.from(allTilesDiv.querySelectorAll('.article-tile')).map(a => a.outerHTML).join('') + allNewHtml);
                                     const allExpanded = Array.from(allTilesDiv.querySelectorAll('.article-tile'));
@@ -236,27 +263,39 @@
                         const existingContent = document.querySelectorAll('.category-section');
                         existingContent.forEach(section => section.remove());
 
-                        const categoriesHtml = Object.entries(data.categorized_articles).map(([cat, articles]) => `
-                            <div class="category-section">
-                                <div class="category-title" onclick="toggleCategory('${cat}')">${cat} <span class="loading-spinner" id="spinner-${cat}" style="display: none;"></span></div>
-                                <div class="tiles" id="category-${cat}" data-last-update="0">
-                                    ${articles.map((article, index) => `
-                                        <div class="article-tile" data-published="${article.published}" data-id="${index}">
-                                            ${article.image !== "svg" ? `<img src="${article.image}" alt="Article Image">` : `
-                                                <svg width="100%" height="150" xmlns="http://www.w3.org/2000/svg">
-                                                    <rect width="100%" height="100%" fill="#e0e0e0"/>
-                                                    <text x="50%" y="50%" text-anchor="middle" dy=".3em" fill="#666">No Image</text>
-                                                </svg>
-                                            `}
-                                            <div class="title"><a href="${article.link}" target="_blank">${article.title}</a></div>
-                                            <div class="description">${article.description}</div>
-                                            <div class="published">Published: ${article.published}</div>
-                                        </div>
-                                    `).join('')}
+                        const categoriesHtml = Object.entries(data.categorized_articles).map(([cat, articles]) => {
+                            const seenKeys = new Set();
+                            const uniqueArticles = articles.filter(article => {
+                                const key = getArticleKey(article);
+                                if (seenKeys.has(key)) {
+                                    console.warn(`Duplicate article skipped in search for ${cat}: ${key}`);
+                                    return false;
+                                }
+                                seenKeys.add(key);
+                                return true;
+                            });
+                            return `
+                                <div class="category-section">
+                                    <div class="category-title" onclick="toggleCategory('${cat}')">${cat} <span class="loading-spinner" id="spinner-${cat}" style="display: none;"></span></div>
+                                    <div class="tiles" id="category-${cat}" data-last-update="0">
+                                        ${uniqueArticles.map((article, index) => `
+                                            <div class="article-tile" data-published="${article.published}" data-id="${index}" data-key="${getArticleKey(article)}">
+                                                ${article.image !== "svg" ? `<img src="${article.image}" alt="Article Image">` : `
+                                                    <svg width="100%" height="150" xmlns="http://www.w3.org/2000/svg">
+                                                        <rect width="100%" height="100%" fill="#e0e0e0"/>
+                                                        <text x="50%" y="50%" text-anchor="middle" dy=".3em" fill="#666">No Image</text>
+                                                    </svg>
+                                                `}
+                                                <div class="title"><a href="${article.link}" target="_blank">${article.title}</a></div>
+                                                <div class="description">${article.description}</div>
+                                                <div class="published">Published: ${article.published}</div>
+                                            </div>
+                                        `).join('')}
+                                    </div>
+                                    <div class="tiles" id="all-${cat}" style="display: none;"></div>
                                 </div>
-                                <div class="tiles" id="all-${cat}" style="display: none;"></div>
-                            </div>
-                        `).join('');
+                            `;
+                        }).join('');
                         document.querySelector('.search-container').insertAdjacentHTML('afterend', categoriesHtml);
                     } else {
                         alert('No results found.');