diff --git a/downstream.go b/downstream.go
index 4420b40..72445ff 100644
--- a/downstream.go
+++ b/downstream.go
@@ -70,6 +70,11 @@ func newDownstreamConn(srv *Server, netConn net.Conn) *downstreamConn {
 				conn.logger.Printf("failed to write message: %v", err)
 			}
 		}
+		if err := conn.net.Close(); err != nil {
+			conn.logger.Printf("failed to close connection: %v", err)
+		} else {
+			conn.logger.Printf("connection closed")
+		}
 	}()
 
 	return conn
@@ -85,7 +90,6 @@ func (c *downstreamConn) prefix() *irc.Prefix {
 
 func (c *downstreamConn) readMessages() error {
 	c.logger.Printf("new connection")
-	defer c.Close()
 
 	for {
 		msg, err := c.irc.ReadMessage()
@@ -108,7 +112,7 @@ func (c *downstreamConn) readMessages() error {
 		}
 	}
 
-	return c.Close()
+	return nil
 }
 
 func (c *downstreamConn) Close() error {
@@ -129,7 +133,7 @@ func (c *downstreamConn) Close() error {
 	close(c.messages)
 	c.closed = true
 
-	return c.net.Close()
+	return nil
 }
 
 func (c *downstreamConn) handleMessage(msg *irc.Message) error {
diff --git a/upstream.go b/upstream.go
index e3bd487..6c6a9a1 100644
--- a/upstream.go
+++ b/upstream.go
@@ -71,6 +71,11 @@ func connectToUpstream(u *user, upstream *Upstream) (*upstreamConn, error) {
 				conn.logger.Printf("failed to write message: %v", err)
 			}
 		}
+		if err := conn.net.Close(); err != nil {
+			conn.logger.Printf("failed to close connection: %v", err)
+		} else {
+			conn.logger.Printf("connection closed")
+		}
 	}()
 
 	return conn, nil
@@ -80,9 +85,6 @@ func (c *upstreamConn) Close() error {
 	if c.closed {
 		return fmt.Errorf("upstream connection already closed")
 	}
-	if err := c.net.Close(); err != nil {
-		return err
-	}
 	close(c.messages)
 	c.closed = true
 	return nil
@@ -330,8 +332,6 @@ func (c *upstreamConn) register() {
 }
 
 func (c *upstreamConn) readMessages() error {
-	defer c.Close()
-
 	for {
 		msg, err := c.irc.ReadMessage()
 		if err == io.EOF {
@@ -345,5 +345,5 @@ func (c *upstreamConn) readMessages() error {
 		}
 	}
 
-	return c.Close()
+	return nil
 }