Merge pull request #20 from aau-network-security/debug-freeze

Fixed deadlock caused by wrong mutex lock order

internal/daemon/agentpool.go

@@ -591,14 +591,15 @@ func (agentLab *AgentLab) updateLabInfo() {
 }
 
 func (agentLab *AgentLab) close() error {
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
 	if agentLab.Conn == nil {
 		return errors.New("connection in agentLab is nil")
 	}
 	agentClient := aproto.NewAgentClient(agentLab.Conn)
 	_, err := agentClient.CloseLab(ctx, &aproto.CloseLabRequest{LabTag: agentLab.LabInfo.Tag})
 	if err != nil {
-		log.Error().Err(err).Msg("error updating lab info")
+		log.Error().Err(err).Msg("error closing lab")
 		return err
 	}
 	return nil

internal/daemon/daemon.go

@@ -362,8 +362,8 @@ func (d *daemon) labExpiryRoutine() {
 	log.Info().Msg("[lab-expiry-routine] starting routine")
 	for {
 		time.Sleep(1 * time.Second)
-		d.eventpool.M.RLock()
-		for _, event := range d.eventpool.Events {
+		events := d.eventpool.GetAllEvents()
+		for _, event := range events {
 			var wg sync.WaitGroup
 			anyLabsClosed := false
 			event.M.RLock()
@@ -406,7 +406,6 @@ func (d *daemon) labExpiryRoutine() {
 				broadCastCommandToEventTeams(event, updateEventInfo)
 			}
 		}
-		d.eventpool.M.RUnlock()
 	}
 }
 

internal/daemon/eventLabs.go

@@ -183,27 +183,30 @@ func (d *daemon) closeLab(c *gin.Context) {
 		return
 	}
 
-	team.M.Lock()
-	defer team.M.Unlock()
-
-	if team.Lab == nil {
+	teamLab := team.GetLab()
+	if teamLab == nil {
 		log.Debug().Str("team", team.Username).Msg("lab not found for team")
 		c.JSON(http.StatusNotFound, APIResponse{Status: "lab not found"})
 		return
 	}
 
-	defer saveState(d.eventpool, d.conf.StatePath)
-
-	event.M.Lock()
-	delete(event.Labs, team.Lab.LabInfo.Tag)
-	event.M.Unlock()
 	if team.Lab.Conn != nil {
-		if err := team.Lab.close(); err != nil {
-			log.Error().Err(err).Str("team", team.Username).Msg("Error closing lab for team")
-		}
+		go func(team *Team, event *Event) {
+			defer func() {
+				event.M.Lock()
+				delete(event.Labs, team.Lab.LabInfo.Tag)
+				event.M.Unlock()
+				team.M.Lock()
+				team.Lab = nil
+				team.M.Unlock()
+				saveState(d.eventpool, d.conf.StatePath)
+				sendCommandToTeam(team, updateTeam)
+			}()
+			if err := team.Lab.close(); err != nil {
+				log.Error().Err(err).Str("team", team.Username).Msg("Error closing lab for team")
+			}
+		}(team, event)
 	}
-	team.Lab = nil
-	sendCommandToTeam(team, updateTeam)
 
 	c.JSON(http.StatusOK, APIResponse{Status: "OK"})
 }
@@ -226,7 +229,9 @@ func (d *daemon) cancelLabConfigurationRequest(c *gin.Context) {
 	}
 
 	team.M.Lock()
-	defer team.M.Unlock()
+	defer func(team *Team) {
+		team.M.Unlock()
+	}(team)
 
 	if team.Status == InQueue {
 		team.Status = Idle