Switch from Adapter (List to Grid) during execution in a RecyclerView

Currently I have a screen where he shows me a list of objects, for this I created a ListAdapter that I will add to my RecyclerView . I would like that when I press a button, the view changes to grid mode, using GridAdapter . So far I try to change the Adapter of my recyclerView, but it tells me No adapter attached; skipping layout . How could I do it?

public class ListAdapter extends RecyclerView.Adapter<ListAdapter.ViewHolder> {
List<Product> items;
Context context;
public ListAdapter(List<Product> items, Context context){
    this.items= items;
    this.context=context;
}
@Override
public ViewHolder onCreateViewHolder(ViewGroup parent, int viewType) {
    View v = LayoutInflater.from(parent.getContext())
            .inflate(R.layout.cardproductlist, parent, false);
    return new ViewHolder(v);
}

@Override
public void onBindViewHolder(ViewHolder holder, int position) {
    Image image = items.get(position).getImage();
    holder.productName.setText(items.get(position).getName());
    String url = "La url que sea"
    Glide.with(context).load(url).diskCacheStrategy(DiskCacheStrategy.ALL).into(holder.productImage);
    holder.cardView.setOnClickListener(new View.OnClickListener() {
        @Override
        public void onClick(View v) {
            //TODO llamar a la ventana de ver el producto
            Intent intent = new Intent(context, ClothsActivity.class);
            intent.putExtra("propia", true);
            v.getContext().startActivity(intent);
        }
    });
}

@Override
public int getItemCount() {
    return items.size();
}

static class ViewHolder extends RecyclerView.ViewHolder {
    TextView productName;
    ImageView productImage;
    CardView cardView;
    RecyclerView    recyclerView;

    public ViewHolder(View v) {
        super(v);
        productName = (TextView) v.findViewById(R.id.productname);
        productImage = (ImageView) v.findViewById(R.id.productimage);
        // recyclerView = (RecyclerView) v.findViewById(R.id.recyclerView);
        cardView = (CardView)v.findViewById(R.id.productcard);
    }
}

public class GridAdapter extends RecyclerView.Adapter<GridAdapter.ViewHolder> {
List<Product> items;
Context context;

public GridAdapter(List<Product> items, Context context){
    this.items= items;
    this.context=context;
}
@Override
public ViewHolder onCreateViewHolder(ViewGroup parent, int viewType) {
    View v = LayoutInflater.from(parent.getContext())
            .inflate(R.layout.cardproduct, parent, false);
    return new ViewHolder(v);
}

@Override
public void onBindViewHolder(final ViewHolder holder, int position) {
    Log.d("AQUI;","ESTAMOS");
   // holder.productImage.setBackgroundResource(R.drawable.planetas2);
    String url = "La url que sea";
    Glide.with(context)
            .load(url).asBitmap()
            .placeholder(R.drawable.avatar)
            .into(new BitmapImageViewTarget(holder.productImage) {
                @Override
                public void onResourceReady(Bitmap drawable, GlideAnimation anim) {
                    super.onResourceReady(drawable, anim);
                }
            });
    holder.cardView.setOnClickListener(new View.OnClickListener() {
        @Override
        public void onClick(View v) {
            //TODO llamar a la ventana de ver el producto
            Intent intent = new Intent(context, ClothsActivity.class);
            intent.putExtra("propia", true);
            v.getContext().startActivity(intent);
        }
    });
}

@Override
public int getItemCount() {
    return items.size();
}

static class ViewHolder extends RecyclerView.ViewHolder {
    TextView productName;
    ImageView productImage;
    CardView cardView;
    RecyclerView    recyclerView;

    public ViewHolder(View v) {
        super(v);
        productImage = (ImageView) v.findViewById(R.id.productimage);
        cardView = (CardView)v.findViewById(R.id.productcard);
    }
}

And to change the Adapter I do the following in a fragment:

  @Override
public void onActivityCreated(Bundle state)  {
    super.onActivityCreated(state);
    Log.d("CREANDO","onActivityCreated");
    listButton = (ImageButton)getView().findViewById(R.id.listOption);
    gridButton = (ImageButton)getView().findViewById(R.id.gridOption);
    gridButton.setVisibility(View.GONE);

    listButton.setOnClickListener(new View.OnClickListener() {
        @Override
        public void onClick(View v) {
           activateList();
        }
    });

    gridButton.setOnClickListener(new View.OnClickListener() {
        @Override
        public void onClick(View v) {
          activateGrid();
        }
    });
}

public void activateList(){
    Log.d("FRAGMENT GALLERY: ", "Activamos lista, numero de elementos: "+products.size());
    listButton.setVisibility(View.GONE);
    gridButton.setVisibility(View.VISIBLE);
    recyclerView.setLayoutManager(new GridLayoutManager(getActivity(), 1));
    ListAdapter listAdapter = new ListAdapter(products,getActivity());
    recyclerView.setAdapter(listAdapter);
    listAdapter.notifyDataSetChanged();

}

public void activateGrid(){
    Log.d("FRAGMENT GALLERY: ", "Activamos grid, numero de elementos: "+products.size());

    listButton.setVisibility(View.VISIBLE);
    gridButton.setVisibility(View.GONE);
    recyclerView.setLayoutManager(new GridLayoutManager(getActivity(), 3));
    GridAdapter gridAdapter = new GridAdapter(products,getActivity());

    recyclerView.setAdapter(gridAdapter);
    gridAdapter.notifyDataSetChanged();
}

Thank you very much everyone.