Improve Active Workers table and recover stale tasks
Backend: - Add _recover_stale_tasks() to reset tasks stuck in 'in_progress' from previous crashes (tasks >5 min old get reset to pending) - Called automatically on startup Frontend: - Fix dark mode colors using CSS variables instead of hardcoded values - Add elapsed time column showing how long task has been running - Add spinning indicator next to package name - Add status badge (Running/Stale?) - Highlight stale tasks (>5 min) in amber - Auto-updates every 5 seconds with existing refresh
This commit is contained in:
@@ -33,6 +33,45 @@ _cache_worker_running: bool = False
|
||||
_dispatcher_thread: Optional[threading.Thread] = None
|
||||
|
||||
|
||||
def _recover_stale_tasks():
|
||||
"""
|
||||
Recover tasks stuck in 'in_progress' state from a previous crash.
|
||||
|
||||
Called on startup to reset tasks that were being processed when
|
||||
the server crashed. Resets them to 'pending' so they can be retried.
|
||||
"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
# Find tasks that have been in_progress for more than 5 minutes
|
||||
# These are likely from a crashed worker
|
||||
stale_threshold = datetime.utcnow() - timedelta(minutes=5)
|
||||
|
||||
stale_count = (
|
||||
db.query(PyPICacheTask)
|
||||
.filter(
|
||||
PyPICacheTask.status == "in_progress",
|
||||
or_(
|
||||
PyPICacheTask.started_at == None,
|
||||
PyPICacheTask.started_at < stale_threshold,
|
||||
),
|
||||
)
|
||||
.update(
|
||||
{
|
||||
"status": "pending",
|
||||
"started_at": None,
|
||||
}
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
if stale_count > 0:
|
||||
logger.warning(f"Recovered {stale_count} stale in_progress tasks from previous crash")
|
||||
except Exception as e:
|
||||
logger.error(f"Error recovering stale tasks: {e}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def init_cache_worker_pool(max_workers: Optional[int] = None):
|
||||
"""
|
||||
Initialize the cache worker pool. Called on app startup.
|
||||
@@ -47,6 +86,9 @@ def init_cache_worker_pool(max_workers: Optional[int] = None):
|
||||
logger.warning("Cache worker pool already initialized")
|
||||
return
|
||||
|
||||
# Recover any stale tasks from previous crash before starting workers
|
||||
_recover_stale_tasks()
|
||||
|
||||
workers = max_workers or settings.PYPI_CACHE_WORKERS
|
||||
_cache_worker_pool = ThreadPoolExecutor(
|
||||
max_workers=workers,
|
||||
|
||||
@@ -280,14 +280,14 @@
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
color: #3b82f6;
|
||||
color: var(--color-primary, #3b82f6);
|
||||
margin-top: 1rem;
|
||||
}
|
||||
|
||||
.pulse-indicator {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
background-color: #3b82f6;
|
||||
background-color: var(--color-primary, #3b82f6);
|
||||
border-radius: 50%;
|
||||
animation: pulse 1.5s ease-in-out infinite;
|
||||
}
|
||||
@@ -304,21 +304,21 @@
|
||||
}
|
||||
|
||||
.jobs-table.active-table {
|
||||
border: 1px solid #3b82f6;
|
||||
border: 1px solid var(--color-primary, #3b82f6);
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.jobs-table.active-table th {
|
||||
background: #eff6ff;
|
||||
color: #1d4ed8;
|
||||
background: var(--bg-tertiary);
|
||||
color: var(--color-primary, #3b82f6);
|
||||
}
|
||||
|
||||
.jobs-table .active-row {
|
||||
background-color: #f0f9ff;
|
||||
background-color: var(--bg-secondary);
|
||||
}
|
||||
|
||||
.jobs-table .active-row:hover {
|
||||
background-color: #e0f2fe;
|
||||
background-color: var(--bg-tertiary);
|
||||
}
|
||||
|
||||
.jobs-table .version-constraint {
|
||||
@@ -327,6 +327,66 @@
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
/* Spinner for active tasks */
|
||||
.spinner {
|
||||
display: inline-block;
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border: 2px solid var(--border-color);
|
||||
border-top-color: var(--color-primary, #3b82f6);
|
||||
border-radius: 50%;
|
||||
animation: spin 1s linear infinite;
|
||||
margin-right: 0.5rem;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
/* Elapsed time */
|
||||
.elapsed-time {
|
||||
font-family: monospace;
|
||||
font-size: 0.9rem;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.elapsed-time.stale {
|
||||
color: #f59e0b;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* Status badges */
|
||||
.status-badge {
|
||||
display: inline-block;
|
||||
padding: 0.2rem 0.5rem;
|
||||
border-radius: 4px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.status-badge.running {
|
||||
background-color: rgba(59, 130, 246, 0.15);
|
||||
color: var(--color-primary, #3b82f6);
|
||||
}
|
||||
|
||||
.status-badge.stale {
|
||||
background-color: rgba(245, 158, 11, 0.15);
|
||||
color: #f59e0b;
|
||||
}
|
||||
|
||||
/* Stale row highlighting */
|
||||
.jobs-table .stale-row {
|
||||
background-color: rgba(245, 158, 11, 0.1);
|
||||
}
|
||||
|
||||
.jobs-table .stale-row:hover {
|
||||
background-color: rgba(245, 158, 11, 0.15);
|
||||
}
|
||||
|
||||
/* Responsive */
|
||||
@media (max-width: 768px) {
|
||||
.status-cards {
|
||||
|
||||
@@ -212,25 +212,43 @@ function AdminJobsPage() {
|
||||
<th>Version</th>
|
||||
<th>Depth</th>
|
||||
<th>Attempt</th>
|
||||
<th>Started</th>
|
||||
<th>Elapsed</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{activeTasks.map((task) => (
|
||||
<tr key={task.id} className="active-row">
|
||||
<td className="package-name">{task.package}</td>
|
||||
<td className="version-constraint">
|
||||
{task.version_constraint || '*'}
|
||||
</td>
|
||||
<td>{task.depth}</td>
|
||||
<td>{task.attempts + 1}</td>
|
||||
<td className="timestamp">
|
||||
{task.started_at
|
||||
? new Date(task.started_at).toLocaleTimeString()
|
||||
: '-'}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
{activeTasks.map((task) => {
|
||||
const elapsed = task.started_at
|
||||
? Math.floor((Date.now() - new Date(task.started_at).getTime()) / 1000)
|
||||
: 0;
|
||||
const minutes = Math.floor(elapsed / 60);
|
||||
const seconds = elapsed % 60;
|
||||
const elapsedStr = minutes > 0
|
||||
? `${minutes}m ${seconds}s`
|
||||
: `${seconds}s`;
|
||||
const isStale = elapsed > 300; // 5 minutes
|
||||
return (
|
||||
<tr key={task.id} className={`active-row ${isStale ? 'stale-row' : ''}`}>
|
||||
<td className="package-name">
|
||||
<span className="spinner"></span>
|
||||
{task.package}
|
||||
</td>
|
||||
<td className="version-constraint">
|
||||
{task.version_constraint || '*'}
|
||||
</td>
|
||||
<td>{task.depth}</td>
|
||||
<td>{task.attempts + 1}</td>
|
||||
<td className={`elapsed-time ${isStale ? 'stale' : ''}`}>
|
||||
{elapsedStr}
|
||||
</td>
|
||||
<td className="status-cell">
|
||||
<span className={`status-badge ${isStale ? 'stale' : 'running'}`}>
|
||||
{isStale ? 'Stale?' : 'Running'}
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user