Improve Active Workers table and recover stale tasks
Backend: - Add _recover_stale_tasks() to reset tasks stuck in 'in_progress' from previous crashes (tasks >5 min old get reset to pending) - Called automatically on startup Frontend: - Fix dark mode colors using CSS variables instead of hardcoded values - Add elapsed time column showing how long task has been running - Add spinning indicator next to package name - Add status badge (Running/Stale?) - Highlight stale tasks (>5 min) in amber - Auto-updates every 5 seconds with existing refresh
This commit is contained in:
@@ -33,6 +33,45 @@ _cache_worker_running: bool = False
|
|||||||
_dispatcher_thread: Optional[threading.Thread] = None
|
_dispatcher_thread: Optional[threading.Thread] = None
|
||||||
|
|
||||||
|
|
||||||
|
def _recover_stale_tasks():
|
||||||
|
"""
|
||||||
|
Recover tasks stuck in 'in_progress' state from a previous crash.
|
||||||
|
|
||||||
|
Called on startup to reset tasks that were being processed when
|
||||||
|
the server crashed. Resets them to 'pending' so they can be retried.
|
||||||
|
"""
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
# Find tasks that have been in_progress for more than 5 minutes
|
||||||
|
# These are likely from a crashed worker
|
||||||
|
stale_threshold = datetime.utcnow() - timedelta(minutes=5)
|
||||||
|
|
||||||
|
stale_count = (
|
||||||
|
db.query(PyPICacheTask)
|
||||||
|
.filter(
|
||||||
|
PyPICacheTask.status == "in_progress",
|
||||||
|
or_(
|
||||||
|
PyPICacheTask.started_at == None,
|
||||||
|
PyPICacheTask.started_at < stale_threshold,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.update(
|
||||||
|
{
|
||||||
|
"status": "pending",
|
||||||
|
"started_at": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
if stale_count > 0:
|
||||||
|
logger.warning(f"Recovered {stale_count} stale in_progress tasks from previous crash")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error recovering stale tasks: {e}")
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
def init_cache_worker_pool(max_workers: Optional[int] = None):
|
def init_cache_worker_pool(max_workers: Optional[int] = None):
|
||||||
"""
|
"""
|
||||||
Initialize the cache worker pool. Called on app startup.
|
Initialize the cache worker pool. Called on app startup.
|
||||||
@@ -47,6 +86,9 @@ def init_cache_worker_pool(max_workers: Optional[int] = None):
|
|||||||
logger.warning("Cache worker pool already initialized")
|
logger.warning("Cache worker pool already initialized")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Recover any stale tasks from previous crash before starting workers
|
||||||
|
_recover_stale_tasks()
|
||||||
|
|
||||||
workers = max_workers or settings.PYPI_CACHE_WORKERS
|
workers = max_workers or settings.PYPI_CACHE_WORKERS
|
||||||
_cache_worker_pool = ThreadPoolExecutor(
|
_cache_worker_pool = ThreadPoolExecutor(
|
||||||
max_workers=workers,
|
max_workers=workers,
|
||||||
|
|||||||
@@ -280,14 +280,14 @@
|
|||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 0.5rem;
|
gap: 0.5rem;
|
||||||
color: #3b82f6;
|
color: var(--color-primary, #3b82f6);
|
||||||
margin-top: 1rem;
|
margin-top: 1rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.pulse-indicator {
|
.pulse-indicator {
|
||||||
width: 8px;
|
width: 8px;
|
||||||
height: 8px;
|
height: 8px;
|
||||||
background-color: #3b82f6;
|
background-color: var(--color-primary, #3b82f6);
|
||||||
border-radius: 50%;
|
border-radius: 50%;
|
||||||
animation: pulse 1.5s ease-in-out infinite;
|
animation: pulse 1.5s ease-in-out infinite;
|
||||||
}
|
}
|
||||||
@@ -304,21 +304,21 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.jobs-table.active-table {
|
.jobs-table.active-table {
|
||||||
border: 1px solid #3b82f6;
|
border: 1px solid var(--color-primary, #3b82f6);
|
||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.jobs-table.active-table th {
|
.jobs-table.active-table th {
|
||||||
background: #eff6ff;
|
background: var(--bg-tertiary);
|
||||||
color: #1d4ed8;
|
color: var(--color-primary, #3b82f6);
|
||||||
}
|
}
|
||||||
|
|
||||||
.jobs-table .active-row {
|
.jobs-table .active-row {
|
||||||
background-color: #f0f9ff;
|
background-color: var(--bg-secondary);
|
||||||
}
|
}
|
||||||
|
|
||||||
.jobs-table .active-row:hover {
|
.jobs-table .active-row:hover {
|
||||||
background-color: #e0f2fe;
|
background-color: var(--bg-tertiary);
|
||||||
}
|
}
|
||||||
|
|
||||||
.jobs-table .version-constraint {
|
.jobs-table .version-constraint {
|
||||||
@@ -327,6 +327,66 @@
|
|||||||
color: var(--text-secondary);
|
color: var(--text-secondary);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Spinner for active tasks */
|
||||||
|
.spinner {
|
||||||
|
display: inline-block;
|
||||||
|
width: 12px;
|
||||||
|
height: 12px;
|
||||||
|
border: 2px solid var(--border-color);
|
||||||
|
border-top-color: var(--color-primary, #3b82f6);
|
||||||
|
border-radius: 50%;
|
||||||
|
animation: spin 1s linear infinite;
|
||||||
|
margin-right: 0.5rem;
|
||||||
|
vertical-align: middle;
|
||||||
|
}
|
||||||
|
|
||||||
|
@keyframes spin {
|
||||||
|
to {
|
||||||
|
transform: rotate(360deg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Elapsed time */
|
||||||
|
.elapsed-time {
|
||||||
|
font-family: monospace;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
color: var(--text-secondary);
|
||||||
|
}
|
||||||
|
|
||||||
|
.elapsed-time.stale {
|
||||||
|
color: #f59e0b;
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Status badges */
|
||||||
|
.status-badge {
|
||||||
|
display: inline-block;
|
||||||
|
padding: 0.2rem 0.5rem;
|
||||||
|
border-radius: 4px;
|
||||||
|
font-size: 0.75rem;
|
||||||
|
font-weight: 500;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-badge.running {
|
||||||
|
background-color: rgba(59, 130, 246, 0.15);
|
||||||
|
color: var(--color-primary, #3b82f6);
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-badge.stale {
|
||||||
|
background-color: rgba(245, 158, 11, 0.15);
|
||||||
|
color: #f59e0b;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Stale row highlighting */
|
||||||
|
.jobs-table .stale-row {
|
||||||
|
background-color: rgba(245, 158, 11, 0.1);
|
||||||
|
}
|
||||||
|
|
||||||
|
.jobs-table .stale-row:hover {
|
||||||
|
background-color: rgba(245, 158, 11, 0.15);
|
||||||
|
}
|
||||||
|
|
||||||
/* Responsive */
|
/* Responsive */
|
||||||
@media (max-width: 768px) {
|
@media (max-width: 768px) {
|
||||||
.status-cards {
|
.status-cards {
|
||||||
|
|||||||
@@ -212,25 +212,43 @@ function AdminJobsPage() {
|
|||||||
<th>Version</th>
|
<th>Version</th>
|
||||||
<th>Depth</th>
|
<th>Depth</th>
|
||||||
<th>Attempt</th>
|
<th>Attempt</th>
|
||||||
<th>Started</th>
|
<th>Elapsed</th>
|
||||||
|
<th>Status</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{activeTasks.map((task) => (
|
{activeTasks.map((task) => {
|
||||||
<tr key={task.id} className="active-row">
|
const elapsed = task.started_at
|
||||||
<td className="package-name">{task.package}</td>
|
? Math.floor((Date.now() - new Date(task.started_at).getTime()) / 1000)
|
||||||
|
: 0;
|
||||||
|
const minutes = Math.floor(elapsed / 60);
|
||||||
|
const seconds = elapsed % 60;
|
||||||
|
const elapsedStr = minutes > 0
|
||||||
|
? `${minutes}m ${seconds}s`
|
||||||
|
: `${seconds}s`;
|
||||||
|
const isStale = elapsed > 300; // 5 minutes
|
||||||
|
return (
|
||||||
|
<tr key={task.id} className={`active-row ${isStale ? 'stale-row' : ''}`}>
|
||||||
|
<td className="package-name">
|
||||||
|
<span className="spinner"></span>
|
||||||
|
{task.package}
|
||||||
|
</td>
|
||||||
<td className="version-constraint">
|
<td className="version-constraint">
|
||||||
{task.version_constraint || '*'}
|
{task.version_constraint || '*'}
|
||||||
</td>
|
</td>
|
||||||
<td>{task.depth}</td>
|
<td>{task.depth}</td>
|
||||||
<td>{task.attempts + 1}</td>
|
<td>{task.attempts + 1}</td>
|
||||||
<td className="timestamp">
|
<td className={`elapsed-time ${isStale ? 'stale' : ''}`}>
|
||||||
{task.started_at
|
{elapsedStr}
|
||||||
? new Date(task.started_at).toLocaleTimeString()
|
</td>
|
||||||
: '-'}
|
<td className="status-cell">
|
||||||
|
<span className={`status-badge ${isStale ? 'stale' : 'running'}`}>
|
||||||
|
{isStale ? 'Stale?' : 'Running'}
|
||||||
|
</span>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
))}
|
);
|
||||||
|
})}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
Reference in New Issue
Block a user