Java Swing: Action Listener Not Working Correctly

For some reason my action listener isn't working correctly. When I add the first number to it it works fine, but when I continue to add numbers it stops working correctly. Any ideas on why this happens would be appreciated!

 import java.awt.*;
import java.awt.event.*;
import javax.swing.*;

public class BabyCalculatorFinal extends JFrame{

private JLabel additionLabel;
private JTextField additionField;
private JButton additionButton;
private JPanel multiplication;
private JLabel multiplicationLabel;
private JTextField multiplicationField;
private JButton multiplicationButton;
private JPanel total;
private JLabel totalLabel;
private JTextField totalField;
private JButton stopButton;

public BabyCalculatorFinal(){
    setDefaultCloseOperation(EXIT_ON_CLOSE);// 1st thing to do
    setName("Baby Calculator Final"); // 2nd thing to do
    setLayout(new GridLayout(3,0)); //sets grid layout for the entire thing with 3 rows
    // Create Action Event
    BabyCalculatorListener listener = new BabyCalculatorListener();
    //Addition 
    //Addition Set Layout
    JPanel addition = new JPanel(new BorderLayout());
    //Addition Features
    additionLabel = new JLabel("Amount to add"); //Create label
    additionField = new JTextField(10);
    additionButton = new JButton("Add");
    //Organize Addition Panel
    addition.add(additionLabel, BorderLayout.WEST);//IMPORTANT FORMAT
    addition.add(additionField, BorderLayout.CENTER);
    addition.add(additionButton, BorderLayout.EAST);
    //Add addition Panel to Frame
    add(addition);
    additionButton.addActionListener(listener);

    //Multiplictation
    //Multiplication Set Layout
    multiplication = new JPanel();
    multiplication.setLayout(new BorderLayout());//Trying a different way of setting the layout
    //Multiplication Features
    multiplicationLabel = new JLabel("Amount to Multiply"); //Create label
    multiplicationField = new JTextField(10);
    multiplicationButton = new JButton("Multiply");
    //Organize Multiplication Panel
    multiplication.add(multiplicationLabel, BorderLayout.WEST);
    multiplication.add(multiplicationField, BorderLayout.CENTER);
    multiplication.add(multiplicationButton, BorderLayout.EAST);
    //Add Multiplication Panel to Frame
    add(multiplication);
    multiplicationButton.addActionListener(listener);
    //Total
    total = new JPanel(new FlowLayout());
    totalLabel = new JLabel("Total");
    totalField = new JTextField();
    totalField.setText("0.0");
    totalField.setEditable(false);
    stopButton = new JButton("Stop");
    total.add(totalLabel);
    total.add(totalField);
    total.add(stopButton);
    //Add Total Panel to Frame
    add(total);
    pack();
    setVisible(true);
}

public static void main(String[] args){
    JFrame myFrame = new BabyCalculatorFinal();
}

public class BabyCalculatorListener implements ActionListener{
    public void actionPerformed(ActionEvent e){
        String totalText = totalField.getText();
        double totalAmount = Double.parseDouble(totalText);
        if (e.getSource() == additionButton){
            String additionText = additionField.getText();
            double addAmount = Double.parseDouble(additionText);
            totalAmount += addAmount;
       }    
       else{
           String multiplicationText = multiplicationField.getText();
           double multiplicationAmount = Double.parseDouble(multiplicationText);
           totalAmount *= multiplicationAmount;
        }
        totalField.setText(totalAmount + "");
    }
}     

}