Io con un bottone creo dinamicamente 2 label e un bottone  ho provate a dare alle due label, e il tasto creato affianco lo stesso NAME : "a", ricerco le label con lo stesso name del bottone :
Public Class Form1
    Dim bottone As New Button
    Dim i As Integer
    Dim j As Integer
    Dim posy As Integer = 0
    Dim posx As Integer = 0
    Dim label As New Label
    Dim btnx As New Button
    Dim idx As Integer
    
    Private Sub Button1_Click(sender As Object, e As EventArgs)
        posy = posy + 40
        posx = 5
        label = New Label
        Button1 = TryCast(sender, Button)
        label.Text = Button1.Text
        label.Name = "a"
        GroupBox1.Controls.Add(label)
        label.Location = New Point(posx, posy)
        posx += (label.Width + 6)
        label = New Label
        label.Text = Button1.Text
        label.Name = "a"
        GroupBox1.Controls.Add(label)
        label.Location = New Point(posx, posy)
        posx += (label.Width + 6)
        btnx = New Button
        btnx.Text = "X"
        btnx.Name = "a"
        GroupBox1.Controls.Add(btnx)
        btnx.Location = New Point(posx, posy)
        AddHandler btnx.Click, AddressOf MyButtonx_Click
    End Sub
    Private Sub MyButtonx_Click(ByVal sender As System.Object, ByVal e As System.EventArgs)
        btnx = TryCast(sender, Button)
        For i = 0 To GroupBox1.Controls.Count Step 1
            If label.Name = btnx.Name Then
                GroupBox1.Controls.Remove(label)
            End If
        Next
    End Sub
End Class