Visual Basic.NET - drag and drop entre 2 listbox

Buenas, tengo 2 listbox y quiero poder arrastrar y soltar elementos entre ambos. Lo hace mas o menos, pero por algo que no se no me deja seleccionar un elemento, es decir solo puedo seleccionar un elemento si aprieto y muevo, si no kiero moverlo del listbox no m deja tenerlo seleccionado, solo puedo arrastrar y seleccionar

Private Sub listbox1_MouseDown(ByVal sender As System.Object, _
ByVal e As System.Windows.Forms.MouseEventArgs) _
Handles ListBox1.MouseDown, ListBox2.MouseDown

Dim aPoint As Point
Dim lbx As ListBox
Dim aIndex As Integer

lbx = CType(sender, ListBox)
aPoint = New Point(e.X, e.Y)
aIndex = lbx.IndexFromPoint(aPoint)
If aIndex >= 0 Then
source = lbx
sourceIndex = aIndex
lbx.DoDragDrop(lbx.Items(aIndex), DragDropEffects.All)
End If
End Sub

Private Sub ListBox2_DragEnter(ByVal sender As Object, ByVal e As System.Windows.Forms.DragEventArgs) Handles ListBox1.DragEnter, ListBox2.DragEnter
If (e.Data.GetDataPresent(DataFormats.Text, True)) Then
e.Effect = DragDropEffects.All
Else
e.Effect = DragDropEffects.None
End If
End Sub


Private Sub listbox1_DragDrop(ByVal sender As System.Object, _
ByVal e As System.Windows.Forms.DragEventArgs) Handles ListBox1.DragDrop, ListBox2.DragDrop

Dim lbx As ListBox

lbx = CType(sender, ListBox)
If Not source Is Nothing Then
source.Items.RemoveAt(sourceIndex)
End If
lbx.Items.Add(e.Data.GetData(DataFormats.Text))
End Sub