Buenas, te paso un ejemplo :
'
Private Declare Function CreateRoundRectRgn Lib "gdi32" _
(ByVal X1 As Long, ByVal Y1 As Long, _
ByVal X2 As Long, ByVal Y2 As Long, _
ByVal X3 As Long, ByVal Y3 As Long) As Long
Private Declare Function SetWindowRgn Lib "user32" _
(ByVal hwnd As Long, ByVal hRgn As Long, _
ByVal bRedraw As Boolean) As Long
Private Sub Form_Load()
Dim lRet As Long
Dim l As Long
Dim Width As Long
Dim Height As Long
Width = Me.Width / Screen.TwipsPerPixelX
Height = Me.Height / Screen.TwipsPerPixelY
lRet = CreateRoundRectRgn(50, 50, Width - 10, Height - 10, 50, 50) ' si subes los valores el efecto sera mayor
l = SetWindowRgn(Me.hwnd, lRet, True)
End Sub
'
Saludos