n.nakhostin
05c7eae8a2
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
509 lines
14 KiB
Go
509 lines
14 KiB
Go
package ted
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// FileProcessingEmailTemplateData represents the data needed for the file processing email template
|
|
type FileProcessingEmailTemplateData struct {
|
|
OJS string
|
|
ProcessedCount int
|
|
SuccessCount int
|
|
Errors []string
|
|
ProcessedAt int64
|
|
CompanyName string
|
|
CompanyLogo string
|
|
CompanyURL string
|
|
SupportEmail string
|
|
}
|
|
|
|
// GenerateFileProcessingEmailTemplate generates a beautiful HTML email template for file processing results
|
|
func GenerateFileProcessingEmailTemplate(data *FileProcessingEmailTemplateData) string {
|
|
// Format the processing date
|
|
processedAt := time.Unix(data.ProcessedAt, 0).Format("January 2, 2006 at 3:04 PM")
|
|
|
|
// Calculate success rate
|
|
successRate := 0.0
|
|
if data.ProcessedCount > 0 {
|
|
successRate = float64(data.SuccessCount) / float64(data.ProcessedCount) * 100
|
|
}
|
|
|
|
// Determine status color and message
|
|
var statusColor, statusMessage string
|
|
if len(data.Errors) == 0 && data.ProcessedCount > 0 {
|
|
statusColor = "#10b981" // Green
|
|
statusMessage = "All files processed successfully"
|
|
} else if data.SuccessCount > 0 {
|
|
statusColor = "#f59e0b" // Amber
|
|
statusMessage = "Partial success with some errors"
|
|
} else {
|
|
statusColor = "#ef4444" // Red
|
|
statusMessage = "Processing failed"
|
|
}
|
|
|
|
// Generate errors section
|
|
var errorsHTML string
|
|
if len(data.Errors) > 0 {
|
|
errorsHTML = `
|
|
<div class="errors-section">
|
|
<h3>Errors Encountered</h3>
|
|
<div class="errors-list">`
|
|
|
|
for i, error := range data.Errors {
|
|
if i >= 10 { // Limit to first 10 errors
|
|
errorsHTML += fmt.Sprintf(`
|
|
<div class="error-item">
|
|
<span class="error-number">%d.</span>
|
|
<span class="error-text">%s</span>
|
|
</div>`, i+1, error)
|
|
}
|
|
}
|
|
|
|
if len(data.Errors) > 10 {
|
|
errorsHTML += fmt.Sprintf(`
|
|
<div class="error-item error-summary">
|
|
<span class="error-text">... and %d more errors</span>
|
|
</div>`, len(data.Errors)-10)
|
|
}
|
|
|
|
errorsHTML += `
|
|
</div>
|
|
</div>`
|
|
}
|
|
|
|
html := `
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>TED File Processing Report - {{COMPANY_NAME}}</title>
|
|
<style>
|
|
* {
|
|
margin: 0;
|
|
padding: 0;
|
|
box-sizing: border-box;
|
|
}
|
|
|
|
body {
|
|
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
|
line-height: 1.6;
|
|
color: #333;
|
|
background-color: #f8fafc;
|
|
}
|
|
|
|
.email-container {
|
|
max-width: 700px;
|
|
margin: 16px auto;
|
|
background-color: #ffffff;
|
|
border-radius: 12px;
|
|
overflow: hidden;
|
|
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
|
|
}
|
|
|
|
.header {
|
|
background: linear-gradient(135deg, #5750F1 0%, #8b5cf6 100%);
|
|
padding: 40px 30px;
|
|
text-align: center;
|
|
color: white;
|
|
}
|
|
|
|
.logo {
|
|
max-width: 200px;
|
|
height: auto;
|
|
margin-bottom: 20px;
|
|
border-radius: 8px;
|
|
}
|
|
|
|
.header h1 {
|
|
font-size: 28px;
|
|
font-weight: 700;
|
|
margin-bottom: 10px;
|
|
text-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
|
|
}
|
|
|
|
.header p {
|
|
font-size: 16px;
|
|
opacity: 0.9;
|
|
font-weight: 300;
|
|
}
|
|
|
|
.content {
|
|
padding: 40px 30px;
|
|
}
|
|
|
|
.status-banner {
|
|
background-color: {{STATUS_COLOR}};
|
|
color: white;
|
|
padding: 20px;
|
|
border-radius: 8px;
|
|
text-align: center;
|
|
margin-bottom: 30px;
|
|
font-weight: 600;
|
|
font-size: 18px;
|
|
}
|
|
|
|
.processing-card {
|
|
background-color: #f8fafc;
|
|
border: 1px solid #e2e8f0;
|
|
border-radius: 12px;
|
|
padding: 30px;
|
|
margin: 30px 0;
|
|
position: relative;
|
|
}
|
|
|
|
.processing-card::before {
|
|
content: '';
|
|
position: absolute;
|
|
top: 0;
|
|
left: 0;
|
|
right: 0;
|
|
height: 4px;
|
|
background: linear-gradient(90deg, #5750F1, #8b5cf6);
|
|
border-radius: 12px 12px 0 0;
|
|
}
|
|
|
|
.ojs-badge {
|
|
background-color: #5750F1;
|
|
color: white;
|
|
padding: 8px 16px;
|
|
border-radius: 20px;
|
|
font-size: 12px;
|
|
font-weight: 600;
|
|
display: inline-block;
|
|
margin-bottom: 20px;
|
|
letter-spacing: 0.5px;
|
|
}
|
|
|
|
.stats-grid {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
gap: 20px;
|
|
margin-bottom: 30px;
|
|
}
|
|
|
|
.stat-item {
|
|
background-color: white;
|
|
padding: 25px;
|
|
border-radius: 12px;
|
|
border: 1px solid #e2e8f0;
|
|
text-align: center;
|
|
position: relative;
|
|
overflow: hidden;
|
|
}
|
|
|
|
.stat-item::before {
|
|
content: '';
|
|
position: absolute;
|
|
top: 0;
|
|
left: 0;
|
|
right: 0;
|
|
height: 3px;
|
|
background: linear-gradient(90deg, #5750F1, #8b5cf6);
|
|
}
|
|
|
|
.stat-number {
|
|
font-size: 32px;
|
|
font-weight: 700;
|
|
color: #1e293b;
|
|
margin-bottom: 8px;
|
|
}
|
|
|
|
.stat-label {
|
|
font-size: 14px;
|
|
color: #64748b;
|
|
font-weight: 600;
|
|
text-transform: uppercase;
|
|
letter-spacing: 0.5px;
|
|
}
|
|
|
|
.success-rate {
|
|
background: linear-gradient(135deg, #10b981, #059669);
|
|
color: white;
|
|
}
|
|
|
|
.success-rate .stat-number {
|
|
color: white;
|
|
}
|
|
|
|
.success-rate .stat-label {
|
|
color: rgba(255, 255, 255, 0.9);
|
|
}
|
|
|
|
.errors-section {
|
|
margin-top: 30px;
|
|
padding-top: 30px;
|
|
border-top: 1px solid #e2e8f0;
|
|
}
|
|
|
|
.errors-section h3 {
|
|
color: #ef4444;
|
|
font-size: 18px;
|
|
font-weight: 600;
|
|
margin-bottom: 15px;
|
|
}
|
|
|
|
.errors-list {
|
|
background-color: #fef2f2;
|
|
border: 1px solid #fecaca;
|
|
border-radius: 8px;
|
|
padding: 20px;
|
|
max-height: 300px;
|
|
overflow-y: auto;
|
|
}
|
|
|
|
.error-item {
|
|
display: flex;
|
|
align-items: flex-start;
|
|
margin-bottom: 12px;
|
|
padding: 12px;
|
|
background-color: white;
|
|
border-radius: 6px;
|
|
border-left: 4px solid #ef4444;
|
|
}
|
|
|
|
.error-item:last-child {
|
|
margin-bottom: 0;
|
|
}
|
|
|
|
.error-number {
|
|
background-color: #ef4444;
|
|
color: white;
|
|
width: 24px;
|
|
height: 24px;
|
|
border-radius: 50%;
|
|
display: flex;
|
|
align-items: center;
|
|
justify-content: center;
|
|
font-size: 12px;
|
|
font-weight: 600;
|
|
margin-right: 12px;
|
|
flex-shrink: 0;
|
|
}
|
|
|
|
.error-text {
|
|
color: #374151;
|
|
font-size: 14px;
|
|
line-height: 1.5;
|
|
}
|
|
|
|
.error-summary {
|
|
background-color: #fef3c7;
|
|
border-left-color: #f59e0b;
|
|
}
|
|
|
|
.error-summary .error-number {
|
|
background-color: #f59e0b;
|
|
}
|
|
|
|
.timestamp {
|
|
text-align: center;
|
|
color: #64748b;
|
|
font-size: 14px;
|
|
margin-top: 30px;
|
|
padding-top: 20px;
|
|
border-top: 1px solid #e2e8f0;
|
|
}
|
|
|
|
.footer {
|
|
background-color: #f8fafc;
|
|
padding: 30px;
|
|
text-align: center;
|
|
border-top: 1px solid #e2e8f0;
|
|
}
|
|
|
|
.footer p {
|
|
color: #64748b;
|
|
font-size: 14px;
|
|
margin-bottom: 10px;
|
|
}
|
|
|
|
.footer a {
|
|
color: #5750F1;
|
|
text-decoration: none;
|
|
}
|
|
|
|
.footer a:hover {
|
|
text-decoration: underline;
|
|
}
|
|
|
|
@media (max-width: 600px) {
|
|
.email-container {
|
|
margin: 16px;
|
|
border-radius: 0;
|
|
}
|
|
.content {
|
|
padding: 20px;
|
|
}
|
|
|
|
.stats-grid {
|
|
grid-template-columns: 1fr;
|
|
gap: 15px;
|
|
}
|
|
|
|
.header {
|
|
padding: 30px 20px;
|
|
}
|
|
|
|
.header h1 {
|
|
font-size: 24px;
|
|
}
|
|
|
|
.stat-item {
|
|
padding: 20px;
|
|
}
|
|
|
|
.stat-number {
|
|
font-size: 28px;
|
|
}
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="email-container">
|
|
<!-- Header -->
|
|
<div class="header">
|
|
<h1>TED File Processing Report</h1>
|
|
<p>Automated processing results from the TED scraper system</p>
|
|
</div>
|
|
|
|
<!-- Content -->
|
|
<div class="content">
|
|
<div class="status-banner">
|
|
{{STATUS_MESSAGE}}
|
|
</div>
|
|
|
|
<div class="processing-card">
|
|
<div class="ojs-badge">OJS: {{OJS}}</div>
|
|
|
|
<div class="stats-grid">
|
|
<div class="stat-item">
|
|
<div class="stat-number">{{PROCESSED_COUNT}}</div>
|
|
<div class="stat-label">Files Processed</div>
|
|
</div>
|
|
|
|
<div class="stat-item">
|
|
<div class="stat-number">{{SUCCESS_COUNT}}</div>
|
|
<div class="stat-label">Successfully Processed</div>
|
|
</div>
|
|
|
|
<div class="stat-item success-rate">
|
|
<div class="stat-number">{{SUCCESS_RATE}}%</div>
|
|
<div class="stat-label">Success Rate</div>
|
|
</div>
|
|
|
|
<div class="stat-item">
|
|
<div class="stat-number">{{ERROR_COUNT}}</div>
|
|
<div class="stat-label">Errors</div>
|
|
</div>
|
|
</div>
|
|
|
|
{{ERRORS_SECTION}}
|
|
|
|
<div class="timestamp">
|
|
<strong>Processed on:</strong> {{PROCESSED_AT}}
|
|
<br>
|
|
<p>This report was automatically generated by the TED Scraper System</p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Footer -->
|
|
<div class="footer">
|
|
<p>This is an automated message from the TED Scraper System</p>
|
|
<p>For support, contact: <a href="mailto:{{SUPPORT_EMAIL}}">{{SUPPORT_EMAIL}}</a></p>
|
|
<p><a href="{{COMPANY_URL}}">{{COMPANY_NAME}}</a></p>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>`
|
|
|
|
// Replace placeholders with actual data
|
|
html = strings.ReplaceAll(html, "{{COMPANY_NAME}}", data.CompanyName)
|
|
html = strings.ReplaceAll(html, "{{OJS}}", data.OJS)
|
|
html = strings.ReplaceAll(html, "{{PROCESSED_COUNT}}", fmt.Sprintf("%d", data.ProcessedCount))
|
|
html = strings.ReplaceAll(html, "{{SUCCESS_COUNT}}", fmt.Sprintf("%d", data.SuccessCount))
|
|
html = strings.ReplaceAll(html, "{{SUCCESS_RATE}}", fmt.Sprintf("%.1f", successRate))
|
|
html = strings.ReplaceAll(html, "{{ERROR_COUNT}}", fmt.Sprintf("%d", len(data.Errors)))
|
|
html = strings.ReplaceAll(html, "{{PROCESSED_AT}}", processedAt)
|
|
html = strings.ReplaceAll(html, "{{STATUS_COLOR}}", statusColor)
|
|
html = strings.ReplaceAll(html, "{{STATUS_MESSAGE}}", statusMessage)
|
|
html = strings.ReplaceAll(html, "{{ERRORS_SECTION}}", errorsHTML)
|
|
html = strings.ReplaceAll(html, "{{SUPPORT_EMAIL}}", data.SupportEmail)
|
|
html = strings.ReplaceAll(html, "{{COMPANY_URL}}", data.CompanyURL)
|
|
|
|
return html
|
|
}
|
|
|
|
// GenerateFileProcessingEmailMessage generates a simple text message for the email
|
|
func GenerateFileProcessingEmailMessage(data *FileProcessingEmailTemplateData) string {
|
|
processedAt := time.Unix(data.ProcessedAt, 0).Format("January 2, 2006 at 3:04 PM")
|
|
|
|
// Calculate success rate
|
|
successRate := 0.0
|
|
if data.ProcessedCount > 0 {
|
|
successRate = float64(data.SuccessCount) / float64(data.ProcessedCount) * 100
|
|
}
|
|
|
|
// Determine status message
|
|
var statusMessage string
|
|
if len(data.Errors) == 0 && data.ProcessedCount > 0 {
|
|
statusMessage = "✅ All files processed successfully"
|
|
} else if data.SuccessCount > 0 {
|
|
statusMessage = "⚠️ Partial success with some errors"
|
|
} else {
|
|
statusMessage = "❌ Processing failed"
|
|
}
|
|
|
|
message := fmt.Sprintf(`
|
|
TED File Processing Report
|
|
|
|
%s
|
|
|
|
OJS: %s
|
|
Files Processed: %d
|
|
Successfully Processed: %d
|
|
Success Rate: %.1f%%
|
|
Errors: %d
|
|
|
|
Processed on: %s
|
|
|
|
`, statusMessage, data.OJS, data.ProcessedCount, data.SuccessCount, successRate, len(data.Errors), processedAt)
|
|
|
|
// Add errors if any
|
|
if len(data.Errors) > 0 {
|
|
message += "Errors encountered:\n"
|
|
for i, error := range data.Errors {
|
|
if i >= 10 { // Limit to first 10 errors
|
|
break
|
|
}
|
|
message += fmt.Sprintf("%d. %s\n", i+1, error)
|
|
}
|
|
|
|
if len(data.Errors) > 10 {
|
|
message += fmt.Sprintf("... and %d more errors\n", len(data.Errors)-10)
|
|
}
|
|
}
|
|
|
|
message += "\nThis report was automatically generated by the TED Scraper System."
|
|
|
|
return message
|
|
}
|
|
|
|
// GenerateFileProcessingEmailSubject generates an appropriate subject line for the email
|
|
func GenerateFileProcessingEmailSubject(data *FileProcessingEmailTemplateData) string {
|
|
var status string
|
|
if len(data.Errors) == 0 && data.ProcessedCount > 0 {
|
|
status = "✅ Success"
|
|
} else if data.SuccessCount > 0 {
|
|
status = "⚠️ Partial Success"
|
|
} else {
|
|
status = "❌ Failed"
|
|
}
|
|
|
|
return fmt.Sprintf("TED Processing Report - OJS %s - %s (%d/%d files)",
|
|
data.OJS, status, data.SuccessCount, data.ProcessedCount)
|
|
}
|