Here is possible approach - to keep selection during drag. Tested with Xcode 11.4 / iOS 13.4.

@GestureState private var dragOffset = CGSize.zero
@State private var selected: Plant? = nil // << your plant type here
var body: some View {

    //Plants Headline
    VStack (spacing: 5) {
        Text("Plants")
            .font(.headline)
            .fontWeight(.light)

        //HStack for Plants Images
        HStack (spacing: 10) {

            //For each that loops through all of the plants inside the plant box
            ForEach(plantBoxViewModel.plants) { plant in

                //Image of the individual plant (obtained via http request to the backend)
                Image(base64String: plant.picture)
                    .resizable()
                    .clipShape(Circle())
                    .overlay(Circle().stroke(Color.white, lineWidth: 2))
                    .offset(x: self.selected == plant ? self.dragOffset.width : 0,
                            y: self.selected == plant ? self.dragOffset.height : 0)
                    .animation(.easeInOut)
                    .aspectRatio(contentMode: .fill)
                    .frame(width: 70, height: 70)
                    .gesture(
                        DragGesture()
                            .updating(self.$dragOffset, body: { (value, state, transaction) in
                                if nil == self.selected {
                                    self.selected = plant
                                }
                                state = value.translation
                            }).onEnded { _ in self.selected = nil }
                )
            }
        }
    }

}