Add TED Scraper Configuration and Cron Scheduler Implementation
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
This commit is contained in:
n.nakhostin
@@ -0,0 +1,508 @@
|
||||
package ted
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FileProcessingEmailTemplateData represents the data needed for the file processing email template
|
||||
type FileProcessingEmailTemplateData struct {
|
||||
OJS string
|
||||
ProcessedCount int
|
||||
SuccessCount int
|
||||
Errors []string
|
||||
ProcessedAt int64
|
||||
CompanyName string
|
||||
CompanyLogo string
|
||||
CompanyURL string
|
||||
SupportEmail string
|
||||
}
|
||||
|
||||
// GenerateFileProcessingEmailTemplate generates a beautiful HTML email template for file processing results
|
||||
func GenerateFileProcessingEmailTemplate(data *FileProcessingEmailTemplateData) string {
|
||||
// Format the processing date
|
||||
processedAt := time.Unix(data.ProcessedAt, 0).Format("January 2, 2006 at 3:04 PM")
|
||||
|
||||
// Calculate success rate
|
||||
successRate := 0.0
|
||||
if data.ProcessedCount > 0 {
|
||||
successRate = float64(data.SuccessCount) / float64(data.ProcessedCount) * 100
|
||||
}
|
||||
|
||||
// Determine status color and message
|
||||
var statusColor, statusMessage string
|
||||
if len(data.Errors) == 0 && data.ProcessedCount > 0 {
|
||||
statusColor = "#10b981" // Green
|
||||
statusMessage = "All files processed successfully"
|
||||
} else if data.SuccessCount > 0 {
|
||||
statusColor = "#f59e0b" // Amber
|
||||
statusMessage = "Partial success with some errors"
|
||||
} else {
|
||||
statusColor = "#ef4444" // Red
|
||||
statusMessage = "Processing failed"
|
||||
}
|
||||
|
||||
// Generate errors section
|
||||
var errorsHTML string
|
||||
if len(data.Errors) > 0 {
|
||||
errorsHTML = `
|
||||
<div class="errors-section">
|
||||
<h3>Errors Encountered</h3>
|
||||
<div class="errors-list">`
|
||||
|
||||
for i, error := range data.Errors {
|
||||
if i >= 10 { // Limit to first 10 errors
|
||||
errorsHTML += fmt.Sprintf(`
|
||||
<div class="error-item">
|
||||
<span class="error-number">%d.</span>
|
||||
<span class="error-text">%s</span>
|
||||
</div>`, i+1, error)
|
||||
}
|
||||
}
|
||||
|
||||
if len(data.Errors) > 10 {
|
||||
errorsHTML += fmt.Sprintf(`
|
||||
<div class="error-item error-summary">
|
||||
<span class="error-text">... and %d more errors</span>
|
||||
</div>`, len(data.Errors)-10)
|
||||
}
|
||||
|
||||
errorsHTML += `
|
||||
</div>
|
||||
</div>`
|
||||
}
|
||||
|
||||
html := `
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>TED File Processing Report - {{COMPANY_NAME}}</title>
|
||||
<style>
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||
line-height: 1.6;
|
||||
color: #333;
|
||||
background-color: #f8fafc;
|
||||
}
|
||||
|
||||
.email-container {
|
||||
max-width: 700px;
|
||||
margin: 16px auto;
|
||||
background-color: #ffffff;
|
||||
border-radius: 12px;
|
||||
overflow: hidden;
|
||||
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.header {
|
||||
background: linear-gradient(135deg, #5750F1 0%, #8b5cf6 100%);
|
||||
padding: 40px 30px;
|
||||
text-align: center;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.logo {
|
||||
max-width: 200px;
|
||||
height: auto;
|
||||
margin-bottom: 20px;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.header h1 {
|
||||
font-size: 28px;
|
||||
font-weight: 700;
|
||||
margin-bottom: 10px;
|
||||
text-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.header p {
|
||||
font-size: 16px;
|
||||
opacity: 0.9;
|
||||
font-weight: 300;
|
||||
}
|
||||
|
||||
.content {
|
||||
padding: 40px 30px;
|
||||
}
|
||||
|
||||
.status-banner {
|
||||
background-color: {{STATUS_COLOR}};
|
||||
color: white;
|
||||
padding: 20px;
|
||||
border-radius: 8px;
|
||||
text-align: center;
|
||||
margin-bottom: 30px;
|
||||
font-weight: 600;
|
||||
font-size: 18px;
|
||||
}
|
||||
|
||||
.processing-card {
|
||||
background-color: #f8fafc;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 12px;
|
||||
padding: 30px;
|
||||
margin: 30px 0;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.processing-card::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
height: 4px;
|
||||
background: linear-gradient(90deg, #5750F1, #8b5cf6);
|
||||
border-radius: 12px 12px 0 0;
|
||||
}
|
||||
|
||||
.ojs-badge {
|
||||
background-color: #5750F1;
|
||||
color: white;
|
||||
padding: 8px 16px;
|
||||
border-radius: 20px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
display: inline-block;
|
||||
margin-bottom: 20px;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.stats-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 20px;
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
|
||||
.stat-item {
|
||||
background-color: white;
|
||||
padding: 25px;
|
||||
border-radius: 12px;
|
||||
border: 1px solid #e2e8f0;
|
||||
text-align: center;
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.stat-item::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
height: 3px;
|
||||
background: linear-gradient(90deg, #5750F1, #8b5cf6);
|
||||
}
|
||||
|
||||
.stat-number {
|
||||
font-size: 32px;
|
||||
font-weight: 700;
|
||||
color: #1e293b;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.stat-label {
|
||||
font-size: 14px;
|
||||
color: #64748b;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.success-rate {
|
||||
background: linear-gradient(135deg, #10b981, #059669);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.success-rate .stat-number {
|
||||
color: white;
|
||||
}
|
||||
|
||||
.success-rate .stat-label {
|
||||
color: rgba(255, 255, 255, 0.9);
|
||||
}
|
||||
|
||||
.errors-section {
|
||||
margin-top: 30px;
|
||||
padding-top: 30px;
|
||||
border-top: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.errors-section h3 {
|
||||
color: #ef4444;
|
||||
font-size: 18px;
|
||||
font-weight: 600;
|
||||
margin-bottom: 15px;
|
||||
}
|
||||
|
||||
.errors-list {
|
||||
background-color: #fef2f2;
|
||||
border: 1px solid #fecaca;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
max-height: 300px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.error-item {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
margin-bottom: 12px;
|
||||
padding: 12px;
|
||||
background-color: white;
|
||||
border-radius: 6px;
|
||||
border-left: 4px solid #ef4444;
|
||||
}
|
||||
|
||||
.error-item:last-child {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.error-number {
|
||||
background-color: #ef4444;
|
||||
color: white;
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
border-radius: 50%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
margin-right: 12px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.error-text {
|
||||
color: #374151;
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.error-summary {
|
||||
background-color: #fef3c7;
|
||||
border-left-color: #f59e0b;
|
||||
}
|
||||
|
||||
.error-summary .error-number {
|
||||
background-color: #f59e0b;
|
||||
}
|
||||
|
||||
.timestamp {
|
||||
text-align: center;
|
||||
color: #64748b;
|
||||
font-size: 14px;
|
||||
margin-top: 30px;
|
||||
padding-top: 20px;
|
||||
border-top: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.footer {
|
||||
background-color: #f8fafc;
|
||||
padding: 30px;
|
||||
text-align: center;
|
||||
border-top: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.footer p {
|
||||
color: #64748b;
|
||||
font-size: 14px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.footer a {
|
||||
color: #5750F1;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.footer a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.email-container {
|
||||
margin: 16px;
|
||||
border-radius: 0;
|
||||
}
|
||||
.content {
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.stats-grid {
|
||||
grid-template-columns: 1fr;
|
||||
gap: 15px;
|
||||
}
|
||||
|
||||
.header {
|
||||
padding: 30px 20px;
|
||||
}
|
||||
|
||||
.header h1 {
|
||||
font-size: 24px;
|
||||
}
|
||||
|
||||
.stat-item {
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.stat-number {
|
||||
font-size: 28px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="email-container">
|
||||
<!-- Header -->
|
||||
<div class="header">
|
||||
<h1>TED File Processing Report</h1>
|
||||
<p>Automated processing results from the TED scraper system</p>
|
||||
</div>
|
||||
|
||||
<!-- Content -->
|
||||
<div class="content">
|
||||
<div class="status-banner">
|
||||
{{STATUS_MESSAGE}}
|
||||
</div>
|
||||
|
||||
<div class="processing-card">
|
||||
<div class="ojs-badge">OJS: {{OJS}}</div>
|
||||
|
||||
<div class="stats-grid">
|
||||
<div class="stat-item">
|
||||
<div class="stat-number">{{PROCESSED_COUNT}}</div>
|
||||
<div class="stat-label">Files Processed</div>
|
||||
</div>
|
||||
|
||||
<div class="stat-item">
|
||||
<div class="stat-number">{{SUCCESS_COUNT}}</div>
|
||||
<div class="stat-label">Successfully Processed</div>
|
||||
</div>
|
||||
|
||||
<div class="stat-item success-rate">
|
||||
<div class="stat-number">{{SUCCESS_RATE}}%</div>
|
||||
<div class="stat-label">Success Rate</div>
|
||||
</div>
|
||||
|
||||
<div class="stat-item">
|
||||
<div class="stat-number">{{ERROR_COUNT}}</div>
|
||||
<div class="stat-label">Errors</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{{ERRORS_SECTION}}
|
||||
|
||||
<div class="timestamp">
|
||||
<strong>Processed on:</strong> {{PROCESSED_AT}}
|
||||
<br>
|
||||
<p>This report was automatically generated by the TED Scraper System</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Footer -->
|
||||
<div class="footer">
|
||||
<p>This is an automated message from the TED Scraper System</p>
|
||||
<p>For support, contact: <a href="mailto:{{SUPPORT_EMAIL}}">{{SUPPORT_EMAIL}}</a></p>
|
||||
<p><a href="{{COMPANY_URL}}">{{COMPANY_NAME}}</a></p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
// Replace placeholders with actual data
|
||||
html = strings.ReplaceAll(html, "{{COMPANY_NAME}}", data.CompanyName)
|
||||
html = strings.ReplaceAll(html, "{{OJS}}", data.OJS)
|
||||
html = strings.ReplaceAll(html, "{{PROCESSED_COUNT}}", fmt.Sprintf("%d", data.ProcessedCount))
|
||||
html = strings.ReplaceAll(html, "{{SUCCESS_COUNT}}", fmt.Sprintf("%d", data.SuccessCount))
|
||||
html = strings.ReplaceAll(html, "{{SUCCESS_RATE}}", fmt.Sprintf("%.1f", successRate))
|
||||
html = strings.ReplaceAll(html, "{{ERROR_COUNT}}", fmt.Sprintf("%d", len(data.Errors)))
|
||||
html = strings.ReplaceAll(html, "{{PROCESSED_AT}}", processedAt)
|
||||
html = strings.ReplaceAll(html, "{{STATUS_COLOR}}", statusColor)
|
||||
html = strings.ReplaceAll(html, "{{STATUS_MESSAGE}}", statusMessage)
|
||||
html = strings.ReplaceAll(html, "{{ERRORS_SECTION}}", errorsHTML)
|
||||
html = strings.ReplaceAll(html, "{{SUPPORT_EMAIL}}", data.SupportEmail)
|
||||
html = strings.ReplaceAll(html, "{{COMPANY_URL}}", data.CompanyURL)
|
||||
|
||||
return html
|
||||
}
|
||||
|
||||
// GenerateFileProcessingEmailMessage generates a simple text message for the email
|
||||
func GenerateFileProcessingEmailMessage(data *FileProcessingEmailTemplateData) string {
|
||||
processedAt := time.Unix(data.ProcessedAt, 0).Format("January 2, 2006 at 3:04 PM")
|
||||
|
||||
// Calculate success rate
|
||||
successRate := 0.0
|
||||
if data.ProcessedCount > 0 {
|
||||
successRate = float64(data.SuccessCount) / float64(data.ProcessedCount) * 100
|
||||
}
|
||||
|
||||
// Determine status message
|
||||
var statusMessage string
|
||||
if len(data.Errors) == 0 && data.ProcessedCount > 0 {
|
||||
statusMessage = "✅ All files processed successfully"
|
||||
} else if data.SuccessCount > 0 {
|
||||
statusMessage = "⚠️ Partial success with some errors"
|
||||
} else {
|
||||
statusMessage = "❌ Processing failed"
|
||||
}
|
||||
|
||||
message := fmt.Sprintf(`
|
||||
TED File Processing Report
|
||||
|
||||
%s
|
||||
|
||||
OJS: %s
|
||||
Files Processed: %d
|
||||
Successfully Processed: %d
|
||||
Success Rate: %.1f%%
|
||||
Errors: %d
|
||||
|
||||
Processed on: %s
|
||||
|
||||
`, statusMessage, data.OJS, data.ProcessedCount, data.SuccessCount, successRate, len(data.Errors), processedAt)
|
||||
|
||||
// Add errors if any
|
||||
if len(data.Errors) > 0 {
|
||||
message += "Errors encountered:\n"
|
||||
for i, error := range data.Errors {
|
||||
if i >= 10 { // Limit to first 10 errors
|
||||
break
|
||||
}
|
||||
message += fmt.Sprintf("%d. %s\n", i+1, error)
|
||||
}
|
||||
|
||||
if len(data.Errors) > 10 {
|
||||
message += fmt.Sprintf("... and %d more errors\n", len(data.Errors)-10)
|
||||
}
|
||||
}
|
||||
|
||||
message += "\nThis report was automatically generated by the TED Scraper System."
|
||||
|
||||
return message
|
||||
}
|
||||
|
||||
// GenerateFileProcessingEmailSubject generates an appropriate subject line for the email
|
||||
func GenerateFileProcessingEmailSubject(data *FileProcessingEmailTemplateData) string {
|
||||
var status string
|
||||
if len(data.Errors) == 0 && data.ProcessedCount > 0 {
|
||||
status = "✅ Success"
|
||||
} else if data.SuccessCount > 0 {
|
||||
status = "⚠️ Partial Success"
|
||||
} else {
|
||||
status = "❌ Failed"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("TED Processing Report - OJS %s - %s (%d/%d files)",
|
||||
data.OJS, status, data.SuccessCount, data.ProcessedCount)
|
||||
}
|
||||
Reference in New Issue
Block a user