diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 2ac79ef4fd2a..020987ea99b9 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -527,7 +527,8 @@ in_pcbdetach(inp)
 	struct inpcbinfo *ipi = inp->inp_pcbinfo;
 
 #ifdef IPSEC
-	ipsec4_delete_pcbpolicy(inp);
+	if (inp->inp_sp != NULL)
+		ipsec4_delete_pcbpolicy(inp);
 #endif /*IPSEC*/
 	inp->inp_gencnt = ++ipi->ipi_gencnt;
 	in_pcbremlists(inp);
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index e8e7c167601c..341260a3db68 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -719,9 +719,8 @@ in6_pcbdetach(inp)
 	struct inpcbinfo *ipi = inp->inp_pcbinfo;
 
 #ifdef IPSEC
-	if (sotoinpcb(so) != 0)
-		key_freeso(so);
-	ipsec6_delete_pcbpolicy(inp);
+	if (inp->in6p_sp != NULL)
+		ipsec6_delete_pcbpolicy(inp);
 #endif /* IPSEC */
 	inp->inp_gencnt = ++ipi->ipi_gencnt;
 	in_pcbremlists(inp);